Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86 Still waiting to schedule task β€˜pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86’ is offline Agent pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-nkv6w-6zsdg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c948ce0b9ca9186705277d5889f47b849ce26301" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-nkv6w" name: "pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=759c6d81-cccb-454e-88c4-7096dadb8b15 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-nkv6w GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-ff467ea3dec646dcb34a434b6d5ddeaf65f47c579492826166e1422a0f0cf5b0 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86 pingcap_tiflow_pull_cdc_integration_test_1728-nkv6w GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build1859572970=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86 bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-test-1728-nkv6w-6zsdg-m4f86 bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 64317 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 1ec028f7649f9416 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Sat May 04 02:56:23 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/model/kv.go, cdc/model/sink.go, cdc/model/sink_test.go, cdc/processor/processor.go, cdc/processor/sinkmanager/manager.go, cdc/processor/sourcemanager/manager.go, cdc/redo/reader/reader.go, cdc/sink/dmlsink/factory/factory.go, cdc/sink/dmlsink/txn/mysql/mysql.go, cdc/sink/dmlsink/txn/mysql/mysql_test.go, cmd/kafka-consumer/main.go, cmd/pulsar-consumer/main.go, cmd/storage-consumer/main.go, errors.toml, pkg/applier/redo.go, pkg/applier/redo_test.go, pkg/errors/cdc_errors.go, pkg/errors/helper.go, tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml, tests/integration_tests/changefeed_dup_error_restart/conf/workload, tests/integration_tests/changefeed_dup_error_restart/run.sh, tests/integration_tests/force_replicate_table/run.sh, tests/integration_tests/open_protocol_handle_key_only/data/data.sql, tests/integration_tests/run_group.sh] [Pipeline] echo diff file not matched: cdc/model/kv.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534-b297eae) 203837440 bytes in 1.58 secs (128814637 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/.git/ .git HEAD is now at b297eae37 check POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master = [up to date] refs/pull/10919/head -> origin/pr/10919/head Previous HEAD position was b297eae37 check HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) 🚧 Checkouting to base SHA:be1553484fe4c03594eabb8d7435c694e5fd7224... HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) βœ… Checked. πŸŽ‰ 🧾 HEAD info: be1553484fe4c03594eabb8d7435c694e5fd7224 be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) 36e9e1bf6 cli(ticdc): allow client authentication to be enabled without tls (#11005) 🚧 Pre-merge heads of pull requests to base SHA: be1553484fe4c03594eabb8d7435c694e5fd7224 ... Updating be1553484..b297eae37 Fast-forward cdc/model/kv.go | 5 + cdc/model/sink.go | 35 ++- cdc/model/sink_test.go | 9 +- cdc/processor/processor.go | 21 +- cdc/processor/sinkmanager/manager.go | 5 + cdc/processor/sourcemanager/manager.go | 66 +++- cdc/redo/reader/reader.go | 21 +- cdc/sink/dmlsink/factory/factory.go | 8 +- cdc/sink/dmlsink/txn/mysql/mysql.go | 89 +++--- cdc/sink/dmlsink/txn/mysql/mysql_test.go | 2 +- cmd/kafka-consumer/main.go | 4 +- cmd/pulsar-consumer/main.go | 17 +- cmd/storage-consumer/main.go | 4 +- errors.toml | 5 + pkg/applier/redo.go | 303 +++++++++++++++++- pkg/applier/redo_test.go | 347 ++++++++++++++++++++- pkg/errors/cdc_errors.go | 4 + pkg/errors/helper.go | 19 ++ .../conf/diff_config.toml | 29 ++ .../changefeed_dup_error_restart/conf/workload | 13 + .../changefeed_dup_error_restart/run.sh | 54 ++++ .../integration_tests/force_replicate_table/run.sh | 4 +- .../open_protocol_handle_key_only/data/data.sql | 46 +-- tests/integration_tests/run_group.sh | 5 +- 24 files changed, 975 insertions(+), 140 deletions(-) create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/workload create mode 100755 tests/integration_tests/changefeed_dup_error_restart/run.sh 🧾 Pre-merged result: b297eae37cdb06fa18bc5388b04d2564a81a0b87 b297eae37 check dd5a6720f fix 4085e5de5 check test βœ… Pre merged πŸŽ‰ βœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-be15534-b297eae already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1213 0 --:--:-- --:--:-- --:--:-- 1242 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1805 0 --:--:-- --:--:-- --:--:-- 1863 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 899 0 --:--:-- --:--:-- --:--:-- 911 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 872 0 --:--:-- --:--:-- --:--:-- 891 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz 2024-05-04 10:56:46 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz [536570515/536570515] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz 2024-05-04 10:56:59 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz [187372022/187372022] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz 2024-05-04 10:57:36 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz [919098782/919098782] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz 2024-05-04 10:58:04 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz [456057803/456057803] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-04 10:58:09 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-04 10:58:12 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-04 10:58:12 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-04 10:58:14 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-04 10:58:18 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-04 10:58:34 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 10:58 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 10:58 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 10:58 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 10:58 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 10:58 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 10:58 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 10:58 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 10:58 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 10:58 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 10:58 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 02:58:37" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/google/uuid v1.6.0 go: downloading golang.org/x/time v0.5.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/google/btree v1.1.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/swaggo/swag v1.16.3 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/robfig/cron v1.2.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:01:35" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:01:44" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:01:52" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:01:52" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:01:52" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:01:52" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:01:52" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:01:52" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 11:07 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 10:58 .. -rwxr-xr-x. 1 jenkins jenkins 220M May 4 11:07 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 4 11:06 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 11:02 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 11:02 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 11:02 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M May 4 11:02 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-54-gb297eae37 Git Commit Hash: b297eae37cdb06fa18bc5388b04d2564a81a0b87 Git Branch: HEAD UTC Build Time: 2024-05-04 03:01:52 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache not saved (binary/pingcap/tiflow/cdc-integration-test/rev-be15534-b297eae already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 11:07 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 10:58 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 11:07 bin -rwxr-xr-x. 1 jenkins jenkins 220M May 4 11:07 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 4 11:06 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 11:02 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 11:02 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 11:02 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 11:07 etc -rwxr-xr-x. 1 jenkins jenkins 17M May 4 11:07 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 11:07 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 11:07 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 11:07 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 4 11:07 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 4 11:07 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 4 11:07 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 4 11:07 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 4 11:07 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 4 11:07 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 4 11:07 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 4 11:07 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 4 11:07 minio -rwxr-xr-x. 1 jenkins jenkins 12M May 4 11:02 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 37M May 4 11:07 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 4 11:07 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 4 11:07 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 4 11:07 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 4 11:07 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 4 11:07 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M May 4 11:07 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 4 11:07 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 11:07 share -rwxr-xr-x. 1 jenkins jenkins 32M May 4 11:07 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M May 4 11:07 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 4 11:07 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 4 11:07 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M May 4 11:07 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 4 11:07 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 91.26 secs (40876238 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G18') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G19') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G20') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G21') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G18') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G19') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G20') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G21') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-90rq7-pgjcc-g3f88 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt Agent pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-l7q3d-gknk9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a44b1ca2b2b6ee9f9a0ddaba09f765b19abedf71" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-l7q3d" name: "pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@15efe97a; decorates RemoteLauncher[hudson.remoting.Channel@45baf80f:JNLP4-connect connection from 10.233.107.246/10.233.107.246:54730] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Agent pingcap-tiflow-pull-cdc-integration-test-1728-90rq7-pgjcc-g3f88 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-90rq7-pgjcc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2e0c157b9af810eebc4a779c0233f25a8c762e6d" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-90rq7" name: "pingcap-tiflow-pull-cdc-integration-test-1728-90rq7-pgjcc-g3f88" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-90rq7-pgjcc-g3f88" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-90rq7-pgjcc-g3f88" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-90rq7-pgjcc-g3f88 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3 [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7348e856; decorates RemoteLauncher[hudson.remoting.Channel@6cd7c4e8:JNLP4-connect connection from 10.233.84.250/10.233.84.250:45262] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] podTemplate [Pipeline] { [Pipeline] node [Pipeline] withEnv [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-9s9lq-mw7bz --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2d8e98cda49dcc92b32e8a19287ee9b74c8d4086" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-9s9lq" name: "pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-078bq-h57sb --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6754cce5dd7b7405f24120402a8662f02fcc0ada" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-078bq" name: "pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-p9h01-h8b43 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8e7ed4f447ae6a8392158235ebc18c12d828c45f" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-p9h01" name: "pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-793vm-h88dj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9b21938f8a6dc8ceb11642ab7dfa44cdbb253aca" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-793vm" name: "pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-q7cw3-fldzm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "784408b8fe9141c58cf888e1cf9f89aa87501707" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-q7cw3" name: "pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-btg0j-ng7bd --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "bfada357bd4cf53de72b04963cde97d2a660ff18" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-btg0j" name: "pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-9jh7r-sz6lx --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e59a101721205ab51be7ae8e71ca4d9c0dd47957" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-9jh7r" name: "pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-nsgc3-qtc3r --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9a75bea5a8129f8df83f043c3493b72b2f69d1c3" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-nsgc3" name: "pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 20.95 secs (178079394 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility changefeed_dup_error_restart kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=6132d7dd-fcbd-4fe0-9e19-1c98ee6773e9 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-l7q3d GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-l7q3d pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b [Pipeline] timeout Timeout set to expire in 40 min Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2795a15c; decorates RemoteLauncher[hudson.remoting.Channel@6dbcf8b:JNLP4-connect connection from 10.233.105.112/10.233.105.112:34516] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@11f45984; decorates RemoteLauncher[hudson.remoting.Channel@a2cfe68:JNLP4-connect connection from 10.233.106.82/10.233.106.82:52828] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@fec83b1; decorates RemoteLauncher[hudson.remoting.Channel@48e2172d:JNLP4-connect connection from 10.233.93.95/10.233.93.95:51850] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@31351028; decorates RemoteLauncher[hudson.remoting.Channel@700e0d42:JNLP4-connect connection from 10.233.88.42/10.233.88.42:37062] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6e70a05a; decorates RemoteLauncher[hudson.remoting.Channel@57561611:JNLP4-connect connection from 10.233.100.24/10.233.100.24:43872] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7e4945ac; decorates RemoteLauncher[hudson.remoting.Channel@7c339de4:JNLP4-connect connection from 10.233.123.112/10.233.123.112:41518] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6600a4c7; decorates RemoteLauncher[hudson.remoting.Channel@f7e91aa:JNLP4-connect connection from 10.233.68.41/10.233.68.41:40618] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@652183d9; decorates RemoteLauncher[hudson.remoting.Channel@b3de6bd:JNLP4-connect connection from 10.233.86.200/10.233.86.200:55954] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-wfls9-dpq2f --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5b2b97b0378770f122c8adf913c9ebb520b4755d" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-wfls9" name: "pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-z4wx0-w924w Agent pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-7mp2h-b4mjm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d1486252b0a1cf82f61bef56994ac632bc347eb4" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-z4wx0" name: "pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" --- value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" apiVersion: "v1" kind: "Pod" metadata: annotations: value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4a4378d36ef43e602d042e6d4de45bf191f5f9f3" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-7mp2h" name: "pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: medium: "" name: "workspace-volume" requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Running on pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-cgbl9-ffvgz Agent pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-w435w-vg0r5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4c87815c05b2df07c8b0d5ca76a0e2ce7c3f7844" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-cgbl9" jenkins/label-digest: "9f5a111cb80eef6e1b1d33758772d34b40e4adc9" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-w435w" name: "pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: name: "pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" memory: "128Mi" cpu: "100m" tty: true volumeMounts: cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" name: "report" resources: limits: memory: "256Mi" cpu: "100m" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" value: "pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" requests: memory: "256Mi" cpu: "100m" volumeMounts: cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" - emptyDir: medium: "" name: "workspace-volume" name: "workspace-volume" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Running on pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Running on pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg Agent pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-zcsxg-0xclk --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "bd694bebd3fc22813f10e77cca0e86b730b94002" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-zcsxg" name: "pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Agent pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-g278r-3f91t --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cb5bca91baf5dabc1f2933d2d6c5fca244f3d8af" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-g278r" name: "pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Agent pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-mdrtm-k4szs --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "48c7e7cef03bbcffdba2e9c88da3a0bec1d19798" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-mdrtm" name: "pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-kk7r7-cnfh3 Agent pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-12kdt-g1k5m --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" --- apiVersion: "v1" kind: "Pod" metadata: annotations: runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "743d7dadb399f2c2ae35ce52678be6c2dbd32e49" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-12kdt" name: "pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1" namespace: "jenkins-tiflow" labels: jenkins/jenkins-jenkins-agent: "true" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: jenkins/label-digest: "e5b00bef6d35a76878248bd37110a0f228a5bbb6" nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-kk7r7" name: "pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: name: "golang" resources: limits: memory: "16Gi" cpu: "6" requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" resources: limits: memory: "16Gi" cpu: "6" requests: cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" memory: "12Gi" cpu: "4" tty: true volumeMounts: name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false name: "report" resources: limits: memory: "256Mi" cpu: "100m" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" name: "report" resources: limits: memory: "256Mi" cpu: "100m" value: "pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1" - name: "JENKINS_NAME" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" value: "pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" value: "pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2" - name: "JENKINS_NAME" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" value: "pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: - emptyDir: medium: "" name: "workspace-volume" requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Running on pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Agent pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-2z0sp-0g62h --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5561b88a26ff3313778c8e3942c1364888f557e6" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-2z0sp" name: "pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Still waiting to schedule task β€˜pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v’ is offline Agent pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1728-0p427-q4gbg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e86976b3902f7124eaa7a20818cc8638fe187b74" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1728-0p427" name: "pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 12.64 secs (295221200 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G01 Run cases: open_protocol_handle_key_only PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=c55125ff-b8fd-4d14-8912-d72cf382b129 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-90rq7-pgjcc-g3f88 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-90rq7 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1728-90rq7-pgjcc-g3f88 pingcap_tiflow_pull_cdc_integration_test_1728-90rq7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-90rq7-pgjcc-g3f88 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Sat May 4 11:09:52 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7c1202f6; decorates RemoteLauncher[hudson.remoting.Channel@1f4ddae3:JNLP4-connect connection from 10.233.90.131/10.233.90.131:43270] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@396c8efc; decorates RemoteLauncher[hudson.remoting.Channel@3f4fca12:JNLP4-connect connection from 10.233.90.97/10.233.90.97:35110] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4a76256f; decorates RemoteLauncher[hudson.remoting.Channel@6e27f5d3:JNLP4-connect connection from 10.233.72.7/10.233.72.7:55140] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@46bd08d0; decorates RemoteLauncher[hudson.remoting.Channel@66ffa45b:JNLP4-connect connection from 10.233.93.13/10.233.93.13:55118] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@485a5475; decorates RemoteLauncher[hudson.remoting.Channel@51e9da35:JNLP4-connect connection from 10.233.107.26/10.233.107.26:54772] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] withCredentials Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@217aab13; decorates RemoteLauncher[hudson.remoting.Channel@33f06a74:JNLP4-connect connection from 10.233.86.245/10.233.86.245:51390] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7ca29a35; decorates RemoteLauncher[hudson.remoting.Channel@772ad29b:JNLP4-connect connection from 10.233.97.181/10.233.97.181:57312] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3e9fb4a4; decorates RemoteLauncher[hudson.remoting.Channel@a1b7dcb:JNLP4-connect connection from 10.233.66.60/10.233.66.60:38878] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] { Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3dd5db7c; decorates RemoteLauncher[hudson.remoting.Channel@51100fb9:JNLP4-connect connection from 10.233.86.237/10.233.86.237:54294] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4ac1aaf9; decorates RemoteLauncher[hudson.remoting.Channel@7daf34d0:JNLP4-connect connection from 10.233.66.226/10.233.66.226:41906] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@47a52944; decorates RemoteLauncher[hudson.remoting.Channel@6eaa2046:JNLP4-connect connection from 10.233.100.171/10.233.100.171:34206] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@13764937; decorates RemoteLauncher[hudson.remoting.Channel@4d73e13b:JNLP4-connect connection from 10.233.100.155/10.233.100.155:49698] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065185240013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:1240, start at 2024-05-04 11:09:54.400504106 +0800 CST m=+5.333457389 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:11:54.410 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:09:54.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:59:54.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 \033[0;36m<<< Run all test success >>>\033[0m Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065185240013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:1240, start at 2024-05-04 11:09:54.400504106 +0800 CST m=+5.333457389 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:11:54.410 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:09:54.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:59:54.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065185080015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:1326, start at 2024-05-04 11:09:54.417111577 +0800 CST m=+5.272503671 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:11:54.427 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:09:54.420 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:59:54.420 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/error.log Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Sat May 4 11:09:59 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.26442646.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:10:02 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/619b11ca-eb99-45f9-b429-4e65da88c316 {"id":"619b11ca-eb99-45f9-b429-4e65da88c316","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41943a98cc 619b11ca-eb99-45f9-b429-4e65da88c316 /tidb/cdc/default/default/upstream/7364976355141674495 {"id":7364976355141674495,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/619b11ca-eb99-45f9-b429-4e65da88c316 {"id":"619b11ca-eb99-45f9-b429-4e65da88c316","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41943a98cc 619b11ca-eb99-45f9-b429-4e65da88c316 /tidb/cdc/default/default/upstream/7364976355141674495 {"id":7364976355141674495,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/619b11ca-eb99-45f9-b429-4e65da88c316 {"id":"619b11ca-eb99-45f9-b429-4e65da88c316","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41943a98cc 619b11ca-eb99-45f9-b429-4e65da88c316 /tidb/cdc/default/default/upstream/7364976355141674495 {"id":7364976355141674495,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 11:10:02 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.27062708.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8400; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 6.57 secs (567552474 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8400 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:10:05 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/516eed60-62e0-460e-bfb2-d0a605219860 {"id":"516eed60-62e0-460e-bfb2-d0a605219860","address":"127.0.0.1:8400","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792202} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f41943ac187 516eed60-62e0-460e-bfb2-d0a605219860 /tidb/cdc/default/default/upstream/7364976362091177379 {"id":7364976362091177379,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/516eed60-62e0-460e-bfb2-d0a605219860 {"id":"516eed60-62e0-460e-bfb2-d0a605219860","address":"127.0.0.1:8400","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792202} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f41943ac187 516eed60-62e0-460e-bfb2-d0a605219860 /tidb/cdc/default/default/upstream/7364976362091177379 {"id":7364976362091177379,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/516eed60-62e0-460e-bfb2-d0a605219860 {"id":"516eed60-62e0-460e-bfb2-d0a605219860","address":"127.0.0.1:8400","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792202} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f41943ac187 516eed60-62e0-460e-bfb2-d0a605219860 /tidb/cdc/default/default/upstream/7364976362091177379 {"id":7364976362091177379,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2768.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306 -c test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/conf/up.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7364976355141674495,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306","create_time":"2024-05-04T11:10:06.059578695+08:00","start_ts":449522488043634689,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522488043634689,"checkpoint_ts":449522488043634689,"checkpoint_time":"2024-05-04 11:10:05.977"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2805.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:4000 -c test-2 --server http://127.0.0.1:8400 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/conf/down.toml Create changefeed successfully! ID: test-2 Info: {"upstream_id":7364976362091177379,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:4000","create_time":"2024-05-04T11:10:07.969965375+08:00","start_ts":449522488540135427,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522488540135427,"checkpoint_ts":449522488540135427,"checkpoint_time":"2024-05-04 11:10:07.871"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 5.88 secs (634660601 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 16.54 secs (225484492 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 6.06 secs (615376875 bytes/sec) [Pipeline] { [Pipeline] cache table tidb_cdc.syncpoint_v1 exists table tidb_cdc.syncpoint_v1 does not exists table bdr_mode.finish_mark exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:10:52 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 22.49 secs (165885117 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 7.18 secs (519184817 bytes/sec) [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0656d8300019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:4036, start at 2024-05-04 11:11:21.657530783 +0800 CST m=+5.256544090 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:21.664 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:21.662 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:21.662 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0656d8300019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:4036, start at 2024-05-04 11:11:21.657530783 +0800 CST m=+5.256544090 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:21.664 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:21.662 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:21.662 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0656d9bc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:4114, start at 2024-05-04 11:11:21.733928239 +0800 CST m=+5.282196466 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:21.740 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:21.711 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:21.711 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 7.49 secs (497977822 bytes/sec) [Pipeline] { [Pipeline] cache [Sat May 4 11:11:24 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.53715373.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:11:27 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 {"id":"3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792285} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419589e4c5 3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 /tidb/cdc/default/default/upstream/7364976732025096533 {"id":7364976732025096533,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 {"id":"3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792285} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419589e4c5 3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 /tidb/cdc/default/default/upstream/7364976732025096533 {"id":7364976732025096533,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 {"id":"3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792285} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419589e4c5 3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 /tidb/cdc/default/default/upstream/7364976732025096533 {"id":7364976732025096533,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 11:11:27 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=1*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.54265428.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:11:30 GMT < Content-Length: 1259 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 {"id":"3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792285} /tidb/cdc/default/__cdc_meta__/capture/acb83268-ff94-486e-8757-477196a59106 {"id":"acb83268-ff94-486e-8757-477196a59106","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419589e4c5 3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 /tidb/cdc/default/__cdc_meta__/owner/22318f419589e4fa acb83268-ff94-486e-8757-477196a59106 /tidb/cdc/default/default/upstream/7364976732025096533 {"id":7364976732025096533,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 {"id":"3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792285} /tidb/cdc/default/__cdc_meta__/capture/acb83268-ff94-486e-8757-477196a59106 {"id":"acb83268-ff94-486e-8757-477196a59106","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419589e4c5 3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 /tidb/cdc/default/__cdc_meta__/owner/22318f419589e4fa acb83268-ff94-486e-8757-477196a59106 /tidb/cdc/default/default/upstream/7364976732025096533 {"id":7364976732025096533,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 {"id":"3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792285} /tidb/cdc/default/__cdc_meta__/capture/acb83268-ff94-486e-8757-477196a59106 {"id":"acb83268-ff94-486e-8757-477196a59106","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419589e4c5 3e7f4868-88d9-40e4-a2cf-2fd4dca5dc98 /tidb/cdc/default/__cdc_meta__/owner/22318f419589e4fa acb83268-ff94-486e-8757-477196a59106 /tidb/cdc/default/default/upstream/7364976732025096533 {"id":7364976732025096533,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table capture_suicide_while_balance_table.t1 not exists for 1-th check, retry later table capture_suicide_while_balance_table.t1 exists table capture_suicide_while_balance_table.t2 not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 10.96 secs (340360725 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh table capture_suicide_while_balance_table.t2 exists table capture_suicide_while_balance_table.t3 not exists for 1-th check, retry later [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G05 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=f49742eb-0fde-4902-a857-0e7e5734f71e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-p9h01 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-p9h01 pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=1b06f2bd-b77e-4a84-b377-ba3564190a14 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-078bq GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242 pingcap_tiflow_pull_cdc_integration_test_1728-078bq GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=66b36a8d-7184-4799-a7ab-0896e162d502 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-793vm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-793vm pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=31881aa7-db49-49cb-ad06-457fa11c1f74 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-q7cw3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q pingcap_tiflow_pull_cdc_integration_test_1728-q7cw3 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=3d2f1e79-afe5-478e-97f9-d620a31d0864 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-btg0j GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-btg0j pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=1ae05d0a-012a-4ba9-b974-cd9ec9542ec2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-9jh7r GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3 pingcap_tiflow_pull_cdc_integration_test_1728-9jh7r GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table capture_suicide_while_balance_table.t3 exists table capture_suicide_while_balance_table.t4 not exists for 1-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc already exists) [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=35ba429e-9b5a-4e15-9298-4e6c54b8575f BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-nsgc3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn pingcap_tiflow_pull_cdc_integration_test_1728-nsgc3 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { table capture_suicide_while_balance_table.t4 exists [Pipeline] // cache [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=9e6af5fb-989f-4129-acf8-329ae0e7437c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-9s9lq GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q pingcap_tiflow_pull_cdc_integration_test_1728-9s9lq GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // dir [Pipeline] } Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] // withCredentials Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] } [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] // timeout [Pipeline] } [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 140 100 35 100 105 364 1093 --:--:-- --:--:-- --:--:-- 1105 { "status": true, "message": "" start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) }lease 22318f419589e4fa revoked check diff successfully start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 3-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:11:48 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06589eb00011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:1286, start at 2024-05-04 11:11:50.718081734 +0800 CST m=+5.277993100 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:50.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:50.700 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:50.700 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06589eb00011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:1286, start at 2024-05-04 11:11:50.718081734 +0800 CST m=+5.277993100 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:50.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:50.700 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:50.700 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658a0f80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:1368, start at 2024-05-04 11:11:50.872457386 +0800 CST m=+5.357776796 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:50.879 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:50.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:50.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658c954001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:1284, start at 2024-05-04 11:11:53.476672808 +0800 CST m=+5.209565421 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:53.483 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:53.479 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:53.479 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658c954001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:1284, start at 2024-05-04 11:11:53.476672808 +0800 CST m=+5.209565421 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:53.483 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:53.479 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:53.479 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658cad8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:1379, start at 2024-05-04 11:11:53.575254353 +0800 CST m=+5.222717391 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:53.584 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:53.577 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:53.577 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658be64000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242, pid:1301, start at 2024-05-04 11:11:52.743779052 +0800 CST m=+5.460889609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:52.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:52.729 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:52.729 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658be64000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242, pid:1301, start at 2024-05-04 11:11:52.743779052 +0800 CST m=+5.460889609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:52.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:52.729 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:52.729 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658cde00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242, pid:1373, start at 2024-05-04 11:11:53.743742386 +0800 CST m=+6.402403824 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:53.750 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:53.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:53.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658b8840003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:1237, start at 2024-05-04 11:11:52.358847994 +0800 CST m=+5.368168740 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:52.366 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:52.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:52.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658b8840003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:1237, start at 2024-05-04 11:11:52.358847994 +0800 CST m=+5.368168740 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:52.366 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:52.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:52.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658b8740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:1310, start at 2024-05-04 11:11:52.373938851 +0800 CST m=+5.334631440 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:52.380 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:52.349 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:52.349 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658bfd40007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:1243, start at 2024-05-04 11:11:52.82817141 +0800 CST m=+5.263958670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:52.835 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:52.821 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:52.821 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658bfd40007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:1243, start at 2024-05-04 11:11:52.82817141 +0800 CST m=+5.263958670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:52.835 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:52.821 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:52.821 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658c22c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:1327, start at 2024-05-04 11:11:53.010835453 +0800 CST m=+5.362240279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:53.020 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:53.020 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:53.020 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2626.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658df000003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn, pid:1239, start at 2024-05-04 11:11:54.817142999 +0800 CST m=+5.296361605 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:54.823 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:54.816 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:54.816 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 11:11:57 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.26222624.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449522516911718401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522516911718401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:11:57 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.26662668.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658df000003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn, pid:1239, start at 2024-05-04 11:11:54.817142999 +0800 CST m=+5.296361605 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:54.823 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:54.816 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:54.816 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658f34c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn, pid:1322, start at 2024-05-04 11:11:56.150810883 +0800 CST m=+6.542303507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:56.157 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:56.115 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:56.115 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2635.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 11:11:57 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.26792681.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Sat May 4 11:11:58 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.27622764.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449522517467987969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522517467987969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658fba00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt, pid:1178, start at 2024-05-04 11:11:56.693502591 +0800 CST m=+5.351053525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:56.701 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:56.697 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:56.697 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658fba00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt, pid:1178, start at 2024-05-04 11:11:56.693502591 +0800 CST m=+5.351053525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:56.701 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:56.697 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:56.697 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065908dc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt, pid:1257, start at 2024-05-04 11:11:57.542490763 +0800 CST m=+6.147931649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:57.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:57.546 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:57.546 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065910480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q, pid:1236, start at 2024-05-04 11:11:57.99047635 +0800 CST m=+6.325739644 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:58.002 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:57.970 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:57.970 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065910480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q, pid:1236, start at 2024-05-04 11:11:57.99047635 +0800 CST m=+6.325739644 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:58.002 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:57.970 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:57.970 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065911c00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q, pid:1320, start at 2024-05-04 11:11:58.070476522 +0800 CST m=+6.329410752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:58.078 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:58.064 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:58.064 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sat May 4 11:11:59 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.26232625.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:00 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f240ce-f340-40e6-a0c2-60b42be8a54c {"id":"16f240ce-f340-40e6-a0c2-60b42be8a54c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196054acc 16f240ce-f340-40e6-a0c2-60b42be8a54c /tidb/cdc/default/default/upstream/7364976861344291250 {"id":7364976861344291250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f240ce-f340-40e6-a0c2-60b42be8a54c {"id":"16f240ce-f340-40e6-a0c2-60b42be8a54c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196054acc 16f240ce-f340-40e6-a0c2-60b42be8a54c /tidb/cdc/default/default/upstream/7364976861344291250 {"id":7364976861344291250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16f240ce-f340-40e6-a0c2-60b42be8a54c {"id":"16f240ce-f340-40e6-a0c2-60b42be8a54c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196054acc 16f240ce-f340-40e6-a0c2-60b42be8a54c /tidb/cdc/default/default/upstream/7364976861344291250 {"id":7364976861344291250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:00 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab70d1de-65e3-4a7f-b2cc-09c2e5e44cc0 {"id":"ab70d1de-65e3-4a7f-b2cc-09c2e5e44cc0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792318} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196046acd ab70d1de-65e3-4a7f-b2cc-09c2e5e44cc0 /tidb/cdc/default/default/upstream/7364976865294745567 {"id":7364976865294745567,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab70d1de-65e3-4a7f-b2cc-09c2e5e44cc0 {"id":"ab70d1de-65e3-4a7f-b2cc-09c2e5e44cc0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792318} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196046acd ab70d1de-65e3-4a7f-b2cc-09c2e5e44cc0 /tidb/cdc/default/default/upstream/7364976865294745567 {"id":7364976865294745567,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab70d1de-65e3-4a7f-b2cc-09c2e5e44cc0 {"id":"ab70d1de-65e3-4a7f-b2cc-09c2e5e44cc0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792318} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196046acd ab70d1de-65e3-4a7f-b2cc-09c2e5e44cc0 /tidb/cdc/default/default/upstream/7364976865294745567 {"id":7364976865294745567,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 695cd283-6c78-4bd3-9217-93f0800c22c0 Info: {"upstream_id":7364976865294745567,"namespace":"default","id":"695cd283-6c78-4bd3-9217-93f0800c22c0","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:12:00.982766377+08:00","start_ts":449522517325643777,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522517325643777,"checkpoint_ts":449522517325643777,"checkpoint_time":"2024-05-04 11:11:57.679"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:00 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93919fa2-9cb1-4fd9-ae5a-baccf0144f14 {"id":"93919fa2-9cb1-4fd9-ae5a-baccf0144f14","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960515d4 93919fa2-9cb1-4fd9-ae5a-baccf0144f14 /tidb/cdc/default/default/upstream/7364976859494443798 {"id":7364976859494443798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93919fa2-9cb1-4fd9-ae5a-baccf0144f14 {"id":"93919fa2-9cb1-4fd9-ae5a-baccf0144f14","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960515d4 93919fa2-9cb1-4fd9-ae5a-baccf0144f14 /tidb/cdc/default/default/upstream/7364976859494443798 {"id":7364976859494443798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93919fa2-9cb1-4fd9-ae5a-baccf0144f14 {"id":"93919fa2-9cb1-4fd9-ae5a-baccf0144f14","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960515d4 93919fa2-9cb1-4fd9-ae5a-baccf0144f14 /tidb/cdc/default/default/upstream/7364976859494443798 {"id":7364976859494443798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2730.out cli changefeed create --start-ts=449522516911718401 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ ***************** properties ***************** "mysql.port"="4000" "mysql.db"="sink_retry" "updateproportion"="0" "mysql.user"="root" "readproportion"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "recordcount"="10" "threadcount"="2" "readallfields"="true" "dotransactions"="false" "operationcount"="0" "insertproportion"="0" "requestdistribution"="uniform" "workload"="core" ********************************************** Run finished, takes 10.417272ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1760.6, Avg(us): 1983, Min(us): 1147, Max(us): 4645, 95th(us): 5000, 99th(us): 5000 [Sat May 4 11:12:00 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.26972699.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: b13d72d1-90fd-4adb-ac1b-22bdffae57b9 Info: {"upstream_id":7364976859494443798,"namespace":"default","id":"b13d72d1-90fd-4adb-ac1b-22bdffae57b9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:12:01.156968929+08:00","start_ts":449522516911718401,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522516911718401,"checkpoint_ts":449522516911718401,"checkpoint_time":"2024-05-04 11:11:56.100"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2638.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:01 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/543dded5-d415-4e35-bd8d-1c3c48df2584 {"id":"543dded5-d415-4e35-bd8d-1c3c48df2584","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792319} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196065f5c 543dded5-d415-4e35-bd8d-1c3c48df2584 /tidb/cdc/default/default/upstream/7364976871667269309 {"id":7364976871667269309,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/543dded5-d415-4e35-bd8d-1c3c48df2584 {"id":"543dded5-d415-4e35-bd8d-1c3c48df2584","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792319} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196065f5c 543dded5-d415-4e35-bd8d-1c3c48df2584 /tidb/cdc/default/default/upstream/7364976871667269309 {"id":7364976871667269309,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/543dded5-d415-4e35-bd8d-1c3c48df2584 {"id":"543dded5-d415-4e35-bd8d-1c3c48df2584","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792319} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196065f5c 543dded5-d415-4e35-bd8d-1c3c48df2584 /tidb/cdc/default/default/upstream/7364976871667269309 {"id":7364976871667269309,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_changefeed_state http://127.0.0.1:2379 b75ec7b8-ae24-469e-9c4f-72e925be2f1c normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=b75ec7b8-ae24-469e-9c4f-72e925be2f1c + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c b75ec7b8-ae24-469e-9c4f-72e925be2f1c -s + info='{ "upstream_id": 7364976871667269309, "namespace": "default", "id": "b75ec7b8-ae24-469e-9c4f-72e925be2f1c", "state": "normal", "checkpoint_tso": 449522518440804355, "checkpoint_time": "2024-05-04 11:12:01.933", "error": null }' + echo '{ "upstream_id": 7364976871667269309, "namespace": "default", "id": "b75ec7b8-ae24-469e-9c4f-72e925be2f1c", "state": "normal", "checkpoint_tso": 449522518440804355, "checkpoint_time": "2024-05-04 11:12:01.933", "error": null }' { "upstream_id": 7364976871667269309, "namespace": "default", "id": "b75ec7b8-ae24-469e-9c4f-72e925be2f1c", "state": "normal", "checkpoint_tso": 449522518440804355, "checkpoint_time": "2024-05-04 11:12:01.933", "error": null } ++ echo '{' '"upstream_id":' 7364976871667269309, '"namespace":' '"default",' '"id":' '"b75ec7b8-ae24-469e-9c4f-72e925be2f1c",' '"state":' '"normal",' '"checkpoint_tso":' 449522518440804355, '"checkpoint_time":' '"2024-05-04' '11:12:01.933",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364976871667269309, '"namespace":' '"default",' '"id":' '"b75ec7b8-ae24-469e-9c4f-72e925be2f1c",' '"state":' '"normal",' '"checkpoint_tso":' 449522518440804355, '"checkpoint_time":' '"2024-05-04' '11:12:01.933",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully [Sat May 4 11:12:02 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.26382640.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:02 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5164889b-c902-4b89-b6a0-5026cfd5c286 {"id":"5164889b-c902-4b89-b6a0-5026cfd5c286","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792320} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960d28cd 5164889b-c902-4b89-b6a0-5026cfd5c286 /tidb/cdc/default/default/upstream/7364976877098508416 {"id":7364976877098508416,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5164889b-c902-4b89-b6a0-5026cfd5c286 {"id":"5164889b-c902-4b89-b6a0-5026cfd5c286","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792320} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960d28cd 5164889b-c902-4b89-b6a0-5026cfd5c286 /tidb/cdc/default/default/upstream/7364976877098508416 {"id":7364976877098508416,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5164889b-c902-4b89-b6a0-5026cfd5c286 {"id":"5164889b-c902-4b89-b6a0-5026cfd5c286","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792320} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960d28cd 5164889b-c902-4b89-b6a0-5026cfd5c286 /tidb/cdc/default/default/upstream/7364976877098508416 {"id":7364976877098508416,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x + tso='449522518457843713 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522518457843713 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:12:03 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.26822684.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:03 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f3cf042-fbdf-4711-9f7e-565851f496e4 {"id":"8f3cf042-fbdf-4711-9f7e-565851f496e4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792321} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960d5ff1 8f3cf042-fbdf-4711-9f7e-565851f496e4 /tidb/cdc/default/default/upstream/7364976867370059755 {"id":7364976867370059755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f3cf042-fbdf-4711-9f7e-565851f496e4 {"id":"8f3cf042-fbdf-4711-9f7e-565851f496e4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792321} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960d5ff1 8f3cf042-fbdf-4711-9f7e-565851f496e4 /tidb/cdc/default/default/upstream/7364976867370059755 {"id":7364976867370059755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f3cf042-fbdf-4711-9f7e-565851f496e4 {"id":"8f3cf042-fbdf-4711-9f7e-565851f496e4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792321} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960d5ff1 8f3cf042-fbdf-4711-9f7e-565851f496e4 /tidb/cdc/default/default/upstream/7364976867370059755 {"id":7364976867370059755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2752.out cli changefeed create --start-ts=449522517467987969 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: 3b4d9162-fa32-4d59-8d65-6c67086b18e9 Info: {"upstream_id":7364976867370059755,"namespace":"default","id":"3b4d9162-fa32-4d59-8d65-6c67086b18e9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:12:04.304559942+08:00","start_ts":449522517467987969,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522517467987969,"checkpoint_ts":449522517467987969,"checkpoint_time":"2024-05-04 11:11:58.222"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:05 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffd03dc1-0a58-42dc-a6f3-19b70bd1c6f2 {"id":"ffd03dc1-0a58-42dc-a6f3-19b70bd1c6f2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792322} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41961786ca ffd03dc1-0a58-42dc-a6f3-19b70bd1c6f2 /tidb/cdc/default/default/upstream/7364976882948699260 {"id":7364976882948699260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffd03dc1-0a58-42dc-a6f3-19b70bd1c6f2 {"id":"ffd03dc1-0a58-42dc-a6f3-19b70bd1c6f2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792322} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41961786ca ffd03dc1-0a58-42dc-a6f3-19b70bd1c6f2 /tidb/cdc/default/default/upstream/7364976882948699260 {"id":7364976882948699260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffd03dc1-0a58-42dc-a6f3-19b70bd1c6f2 {"id":"ffd03dc1-0a58-42dc-a6f3-19b70bd1c6f2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792322} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41961786ca ffd03dc1-0a58-42dc-a6f3-19b70bd1c6f2 /tidb/cdc/default/default/upstream/7364976882948699260 {"id":7364976882948699260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 2-th time, retry later ***************** properties ***************** "readproportion"="0" "threadcount"="10" "mysql.host"="127.0.0.1" "recordcount"="50" "mysql.db"="consistent_replicate_ddl" "requestdistribution"="uniform" "scanproportion"="0" "dotransactions"="false" "workload"="core" "updateproportion"="0" "insertproportion"="0" "operationcount"="0" "readallfields"="true" "mysql.port"="4000" "fieldcount"="3" "mysql.user"="root" ********************************************** Run finished, takes 14.131292ms INSERT - Takes(s): 0.0, Count: 50, OPS: 7901.5, Avg(us): 2616, Min(us): 1038, Max(us): 8030, 95th(us): 8000, 99th(us): 9000 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 19.88 secs (187612339 bytes/sec) [Pipeline] { [Pipeline] cache + set +x table consistent_replicate_ddl.usertable1 not exists for 1-th check, retry later check diff successfully check_safepoint_forward http://127.0.0.1:2379 7364976861344291250 449522519258693635 449522518052831235 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:06 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5ee4a444-d300-4667-95f9-dee4f89e41e9 {"id":"5ee4a444-d300-4667-95f9-dee4f89e41e9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41961837d3 5ee4a444-d300-4667-95f9-dee4f89e41e9 /tidb/cdc/default/default/upstream/7364976885563358859 {"id":7364976885563358859,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5ee4a444-d300-4667-95f9-dee4f89e41e9 {"id":"5ee4a444-d300-4667-95f9-dee4f89e41e9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41961837d3 5ee4a444-d300-4667-95f9-dee4f89e41e9 /tidb/cdc/default/default/upstream/7364976885563358859 {"id":7364976885563358859,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5ee4a444-d300-4667-95f9-dee4f89e41e9 {"id":"5ee4a444-d300-4667-95f9-dee4f89e41e9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41961837d3 5ee4a444-d300-4667-95f9-dee4f89e41e9 /tidb/cdc/default/default/upstream/7364976885563358859 {"id":7364976885563358859,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2741.out cli changefeed create --start-ts=449522518457843713 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ run task successfully Create changefeed successfully! ID: 43a031a3-712f-4886-beb7-621ceece75c3 Info: {"upstream_id":7364976885563358859,"namespace":"default","id":"43a031a3-712f-4886-beb7-621ceece75c3","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:12:07.008185201+08:00","start_ts":449522518457843713,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522518457843713,"checkpoint_ts":449522518457843713,"checkpoint_time":"2024-05-04 11:12:01.998"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check_changefeed_state http://127.0.0.1:2379 2a7123bf-b5e6-4af7-aba9-dedf1574a5e2 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=2a7123bf-b5e6-4af7-aba9-dedf1574a5e2 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 2a7123bf-b5e6-4af7-aba9-dedf1574a5e2 -s check diff successfully + info='{ "upstream_id": 7364976861344291250, "namespace": "default", "id": "2a7123bf-b5e6-4af7-aba9-dedf1574a5e2", "state": "stopped", "checkpoint_tso": 449522519521099781, "checkpoint_time": "2024-05-04 11:12:06.054", "error": null }' + echo '{ "upstream_id": 7364976861344291250, "namespace": "default", "id": "2a7123bf-b5e6-4af7-aba9-dedf1574a5e2", "state": "stopped", "checkpoint_tso": 449522519521099781, "checkpoint_time": "2024-05-04 11:12:06.054", "error": null }' { "upstream_id": 7364976861344291250, "namespace": "default", "id": "2a7123bf-b5e6-4af7-aba9-dedf1574a5e2", "state": "stopped", "checkpoint_tso": 449522519521099781, "checkpoint_time": "2024-05-04 11:12:06.054", "error": null } ++ echo '{' '"upstream_id":' 7364976861344291250, '"namespace":' '"default",' '"id":' '"2a7123bf-b5e6-4af7-aba9-dedf1574a5e2",' '"state":' '"stopped",' '"checkpoint_tso":' 449522519521099781, '"checkpoint_time":' '"2024-05-04' '11:12:06.054",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7364976861344291250, '"namespace":' '"default",' '"id":' '"2a7123bf-b5e6-4af7-aba9-dedf1574a5e2",' '"state":' '"stopped",' '"checkpoint_tso":' 449522519521099781, '"checkpoint_time":' '"2024-05-04' '11:12:06.054",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7364976861344291250 wait process cdc.test exit for 1-th time... table consistent_replicate_ddl.usertable1 not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:12:08 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable1 not exists for 3-th check, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 2a7123bf-b5e6-4af7-aba9-dedf1574a5e2 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=2a7123bf-b5e6-4af7-aba9-dedf1574a5e2 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 2a7123bf-b5e6-4af7-aba9-dedf1574a5e2 -s + info='{ "upstream_id": 7364976861344291250, "namespace": "default", "id": "2a7123bf-b5e6-4af7-aba9-dedf1574a5e2", "state": "normal", "checkpoint_tso": 449522519521099781, "checkpoint_time": "2024-05-04 11:12:06.054", "error": null }' + echo '{ "upstream_id": 7364976861344291250, "namespace": "default", "id": "2a7123bf-b5e6-4af7-aba9-dedf1574a5e2", "state": "normal", "checkpoint_tso": 449522519521099781, "checkpoint_time": "2024-05-04 11:12:06.054", "error": null }' { "upstream_id": 7364976861344291250, "namespace": "default", "id": "2a7123bf-b5e6-4af7-aba9-dedf1574a5e2", "state": "normal", "checkpoint_tso": 449522519521099781, "checkpoint_time": "2024-05-04 11:12:06.054", "error": null } ++ echo '{' '"upstream_id":' 7364976861344291250, '"namespace":' '"default",' '"id":' '"2a7123bf-b5e6-4af7-aba9-dedf1574a5e2",' '"state":' '"normal",' '"checkpoint_tso":' 449522519521099781, '"checkpoint_time":' '"2024-05-04' '11:12:06.054",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364976861344291250, '"namespace":' '"default",' '"id":' '"2a7123bf-b5e6-4af7-aba9-dedf1574a5e2",' '"state":' '"normal",' '"checkpoint_tso":' 449522519521099781, '"checkpoint_time":' '"2024-05-04' '11:12:06.054",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7364976861344291250 449522519521099780 449522519521099781 run task successfully check_changefeed_state http://127.0.0.1:2379 2a7123bf-b5e6-4af7-aba9-dedf1574a5e2 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=2a7123bf-b5e6-4af7-aba9-dedf1574a5e2 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 2a7123bf-b5e6-4af7-aba9-dedf1574a5e2 -s + info='{ "upstream_id": 7364976861344291250, "namespace": "default", "id": "2a7123bf-b5e6-4af7-aba9-dedf1574a5e2", "state": "stopped", "checkpoint_tso": 449522519521099781, "checkpoint_time": "2024-05-04 11:12:06.054", "error": null }' + echo '{ "upstream_id": 7364976861344291250, "namespace": "default", "id": "2a7123bf-b5e6-4af7-aba9-dedf1574a5e2", "state": "stopped", "checkpoint_tso": 449522519521099781, "checkpoint_time": "2024-05-04 11:12:06.054", "error": null }' { "upstream_id": 7364976861344291250, "namespace": "default", "id": "2a7123bf-b5e6-4af7-aba9-dedf1574a5e2", "state": "stopped", "checkpoint_tso": 449522519521099781, "checkpoint_time": "2024-05-04 11:12:06.054", "error": null } ++ echo '{' '"upstream_id":' 7364976861344291250, '"namespace":' '"default",' '"id":' '"2a7123bf-b5e6-4af7-aba9-dedf1574a5e2",' '"state":' '"stopped",' '"checkpoint_tso":' 449522519521099781, '"checkpoint_time":' '"2024-05-04' '11:12:06.054",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7364976861344291250, '"namespace":' '"default",' '"id":' '"2a7123bf-b5e6-4af7-aba9-dedf1574a5e2",' '"state":' '"stopped",' '"checkpoint_tso":' 449522519521099781, '"checkpoint_time":' '"2024-05-04' '11:12:06.054",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 0662479d-b666-4bfc-a760-71916ced6471 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=0662479d-b666-4bfc-a760-71916ced6471 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 0662479d-b666-4bfc-a760-71916ced6471 -s + info='{ "upstream_id": 7364976861344291250, "namespace": "default", "id": "0662479d-b666-4bfc-a760-71916ced6471", "state": "normal", "checkpoint_tso": 449522521264095233, "checkpoint_time": "2024-05-04 11:12:12.703", "error": null }' + echo '{ "upstream_id": 7364976861344291250, "namespace": "default", "id": "0662479d-b666-4bfc-a760-71916ced6471", "state": "normal", "checkpoint_tso": 449522521264095233, "checkpoint_time": "2024-05-04 11:12:12.703", "error": null }' { "upstream_id": 7364976861344291250, "namespace": "default", "id": "0662479d-b666-4bfc-a760-71916ced6471", "state": "normal", "checkpoint_tso": 449522521264095233, "checkpoint_time": "2024-05-04 11:12:12.703", "error": null } ++ echo '{' '"upstream_id":' 7364976861344291250, '"namespace":' '"default",' '"id":' '"0662479d-b666-4bfc-a760-71916ced6471",' '"state":' '"normal",' '"checkpoint_tso":' 449522521264095233, '"checkpoint_time":' '"2024-05-04' '11:12:12.703",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364976861344291250, '"namespace":' '"default",' '"id":' '"0662479d-b666-4bfc-a760-71916ced6471",' '"state":' '"normal",' '"checkpoint_tso":' 449522521264095233, '"checkpoint_time":' '"2024-05-04' '11:12:12.703",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7364976861344291250 table consistent_replicate_ddl.usertable1 exists table consistent_replicate_ddl.usertable2 not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 6.68 secs (558173147 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable2 not exists for 2-th check, retry later table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later run task successfully Changefeed remove successfully. ID: 2a7123bf-b5e6-4af7-aba9-dedf1574a5e2 CheckpointTs: 449522519521099781 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_forward http://127.0.0.1:2379 7364976861344291250 449522519521099780 449522521264095233 449522519521099781 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065a2638000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:6847, start at 2024-05-04 11:12:15.767788385 +0800 CST m=+5.517075223 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:15.773 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:15.758 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:15.758 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully Changefeed remove successfully. ID: 0662479d-b666-4bfc-a760-71916ced6471 CheckpointTs: 449522522404683782 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_cleared http://127.0.0.1:2379 7364976861344291250 run task successfully table consistent_replicate_ddl.usertable2 exists table consistent_replicate_ddl.usertable3 not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065a2638000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:6847, start at 2024-05-04 11:12:15.767788385 +0800 CST m=+5.517075223 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:15.773 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:15.758 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:15.758 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065a2f700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:6929, start at 2024-05-04 11:12:16.377982567 +0800 CST m=+6.047610566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:16.384 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:16.348 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:16.348 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored table foreign_key.finish_mark not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... table foreign_key.finish_mark not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:12:19 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> table consistent_replicate_ddl.usertable3 not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table row_format.finish_mark not exists for 1-th check, retry later table row_format.finish_mark not exists for 2-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.8206.out cli tso query --pd=http://127.0.0.1:2379 table foreign_key.finish_mark not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table consistent_replicate_ddl.usertable3 exists table consistent_replicate_ddl.usertable_bak not exists for 1-th check, retry later table row_format.finish_mark not exists for 3-th check, retry later + set +x + tso='449522523480522753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522523480522753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:12:22 CST 2024] <<<<<< START cdc server in syncpoint case >>>>>> table test.finish_mark exists check table exists success check diff successfully + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.82418243.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... Verifying downstream PD is started... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table consistent_replicate_ddl.usertable_bak not exists for 2-th check, retry later table foreign_key.finish_mark not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:12:24 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> table row_format.finish_mark not exists for 4-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:25 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/367e95bb-b582-45c1-8a0d-455a25346448 {"id":"367e95bb-b582-45c1-8a0d-455a25346448","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792343} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41965e2be2 367e95bb-b582-45c1-8a0d-455a25346448 /tidb/cdc/default/default/upstream/7364976966983100398 {"id":7364976966983100398,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/367e95bb-b582-45c1-8a0d-455a25346448 {"id":"367e95bb-b582-45c1-8a0d-455a25346448","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792343} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41965e2be2 367e95bb-b582-45c1-8a0d-455a25346448 /tidb/cdc/default/default/upstream/7364976966983100398 {"id":7364976966983100398,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/367e95bb-b582-45c1-8a0d-455a25346448 {"id":"367e95bb-b582-45c1-8a0d-455a25346448","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792343} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41965e2be2 367e95bb-b582-45c1-8a0d-455a25346448 /tidb/cdc/default/default/upstream/7364976966983100398 {"id":7364976966983100398,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.8297.out cli changefeed create --start-ts=449522523480522753 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml table foreign_key.finish_mark not exists for 5-th check, retry later table consistent_replicate_ddl.usertable_bak exists Create changefeed successfully! ID: 2e160851-6e16-47bd-904d-26e47dba7cac Info: {"upstream_id":7364976966983100398,"namespace":"default","id":"2e160851-6e16-47bd-904d-26e47dba7cac","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:12:26.165265679+08:00","start_ts":449522523480522753,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522523480522753,"checkpoint_ts":449522523480522753,"checkpoint_time":"2024-05-04 11:12:21.158"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table row_format.finish_mark not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 6-th check, retry later table row_format.finish_mark not exists for 6-th check, retry later + set +x ***************** properties ***************** "recordcount"="10" "dotransactions"="false" "mysql.user"="root" "mysql.port"="4000" "readproportion"="0" "threadcount"="2" "requestdistribution"="uniform" "workload"="core" "scanproportion"="0" "readallfields"="true" "updateproportion"="0" "insertproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="testSync" "operationcount"="0" ********************************************** Run finished, takes 9.558242ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1847.5, Avg(us): 1824, Min(us): 1090, Max(us): 4038, 95th(us): 5000, 99th(us): 5000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 12.51 secs (298143993 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table foreign_key.finish_mark not exists for 7-th check, retry later table row_format.finish_mark not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... table foreign_key.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... wait process cdc.test exit for 3-th time... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table row_format.finish_mark exists check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065b22b4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:3944, start at 2024-05-04 11:12:31.931550095 +0800 CST m=+5.339481439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:31.939 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:31.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:31.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:12:32 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.28802882.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server2 --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table foreign_key.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:12:34 CST 2024] <<<<<< run test case row_format success! >>>>>> wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065b22b4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:3944, start at 2024-05-04 11:12:31.931550095 +0800 CST m=+5.339481439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:31.939 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:31.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:31.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065b22ac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:4015, start at 2024-05-04 11:12:31.945174967 +0800 CST m=+5.300975759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:31.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:31.915 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:31.915 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:12:35 CST 2024] <<<<<< run test case foreign_key success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {UpstreamID:7364976882948699260 Namespace:default ID:5f56fbd6-928b-4d89-8ab4-67022ab424f7 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:12:05.32333554 +0800 CST StartTs:449522519302733827 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003270f30 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522519328948226} {CheckpointTs:449522527049089027 MinTableBarrierTs:449522527049089027 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/deedb415-ddef-4f60-93a3-476f4350718e {"id":"deedb415-ddef-4f60-93a3-476f4350718e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792353} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419617881c deedb415-ddef-4f60-93a3-476f4350718e /tidb/cdc/default/default/changefeed/info/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {"upstream-id":7364976882948699260,"namespace":"default","changefeed-id":"5f56fbd6-928b-4d89-8ab4-67022ab424f7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:12:05.32333554+08:00","start-ts":449522519302733827,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522519328948226} /tidb/cdc/default/default/changefeed/status/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {"checkpoint-ts":449522527049089027,"min-table-barrier-ts":449522527049089027,"admin-job-type":0} /tidb/cdc/default/default/task/position/deedb415-ddef-4f60-93a3-476f4350718e/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976882948699260 {"id":7364976882948699260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {UpstreamID:7364976882948699260 Namespace:default ID:5f56fbd6-928b-4d89-8ab4-67022ab424f7 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:12:05.32333554 +0800 CST StartTs:449522519302733827 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003270f30 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522519328948226} {CheckpointTs:449522527049089027 MinTableBarrierTs:449522527049089027 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/deedb415-ddef-4f60-93a3-476f4350718e {"id":"deedb415-ddef-4f60-93a3-476f4350718e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792353} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419617881c deedb415-ddef-4f60-93a3-476f4350718e /tidb/cdc/default/default/changefeed/info/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {"upstream-id":7364976882948699260,"namespace":"default","changefeed-id":"5f56fbd6-928b-4d89-8ab4-67022ab424f7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:12:05.32333554+08:00","start-ts":449522519302733827,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck+ grep -q 'failed to get info:' -duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522519328948226} /tidb/cdc/default/default/changefeed/status/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {"checkpoint-ts":449522527049089027,"min-table-barrier-ts":449522527049089027,"admin-job-type":0} /tidb/cdc/default/default/task/position/deedb415-ddef-4f60-93a3-476f4350718e/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976882948699260 {"id":7364976882948699260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {UpstreamID:7364976882948699260 Namespace:default ID:5f56fbd6-928b-4d89-8ab4-67022ab424f7 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:12:05.32333554 +0800 CST StartTs:449522519302733827 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003270f30 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522519328948226} {CheckpointTs:449522527049089027 MinTableBarrierTs:449522527049089027 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449522527049089027, checkpointTs: 449522527049089027, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/deedb415-ddef-4f60-93a3-476f4350718e {"id":"deedb415-ddef-4f60-93a3-476f4350718e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792353} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419617881c deedb415-ddef-4f60-93a3-476f4350718e /tidb/cdc/default/default/changefeed/info/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {"upstream-id":7364976882948699260,"namespace":"default","changefeed-id":"5f56fbd6-928b-4d89-8ab4-67022ab424f7","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:12:05.32333554+08:00","start-ts":449522519302733827,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522519328948226} /tidb/cdc/default/default/changefeed/status/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {"checkpoint-ts":449522527049089027,"min-table-barrier-ts":449522527049089027,"admin-job-type":0} /tidb/cdc/default/default/task/position/deedb415-ddef-4f60-93a3-476f4350718e/5f56fbd6-928b-4d89-8ab4-67022ab424f7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976882948699260 {"id":7364976882948699260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Sat May 4 11:12:36 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.53265328.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "recordcount"="10" "operationcount"="0" "mysql.user"="root" "readallfields"="true" "dotransactions"="false" "scanproportion"="0" "updateproportion"="0" "readproportion"="0" "threadcount"="2" "insertproportion"="0" "mysql.port"="4000" "mysql.db"="sink_retry" "workload"="core" ********************************************** Run finished, takes 4.289696ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3374.2, Avg(us): 742, Min(us): 505, Max(us): 1697, 95th(us): 2000, 99th(us): 2000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:40 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/541ff165-b6de-494f-93dd-0252a23b1021 {"id":"541ff165-b6de-494f-93dd-0252a23b1021","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792357} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41969fd6d1 541ff165-b6de-494f-93dd-0252a23b1021 /tidb/cdc/default/default/upstream/7364977027650215022 {"id":7364977027650215022,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/541ff165-b6de-494f-93dd-0252a23b1021 {"id":"541ff165-b6de-494f-93dd-0252a23b1021","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792357} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41969fd6d1 541ff165-b6de-494f-93dd-0252a23b1021 /tidb/cdc/default/default/upstream/7364977027650215022 {"id":7364977027650215022,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/541ff165-b6de-494f-93dd-0252a23b1021 {"id":"541ff165-b6de-494f-93dd-0252a23b1021","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792357} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41969fd6d1 541ff165-b6de-494f-93dd-0252a23b1021 /tidb/cdc/default/default/upstream/7364977027650215022 {"id":7364977027650215022,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "operationcount"="0" "mysql.host"="127.0.0.1" "workload"="core" "mysql.user"="root" "readproportion"="0" "scanproportion"="0" "readallfields"="true" "insertproportion"="0" "threadcount"="4" "mysql.port"="4000" "dotransactions"="false" "requestdistribution"="uniform" "recordcount"="50" "mysql.db"="changefeed_reconstruct" "updateproportion"="0" ********************************************** Run finished, takes 22.088198ms INSERT - Takes(s): 0.0, Count: 48, OPS: 2791.0, Avg(us): 1759, Min(us): 1167, Max(us): 5016, 95th(us): 5000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:12:40 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 12.40 secs (300916348 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully wait process 5331 exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process 5331 exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bccf80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:4536, start at 2024-05-04 11:12:42.832211169 +0800 CST m=+5.326803762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:42.842 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:42.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:42.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bccf80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:4536, start at 2024-05-04 11:12:42.832211169 +0800 CST m=+5.326803762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:42.842 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:42.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:42.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bcce40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:4617, start at 2024-05-04 11:12:42.836719122 +0800 CST m=+5.282429854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:42.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:42.809 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:42.809 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5331) - No such process wait process 5331 exit for 3-th time... process 5331 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sat May 4 11:12:46 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.56075609.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Sat May 4 11:12:47 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.59485950.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/99f1d090-d56a-4290-9604-dc9ecaf9b879 {UpstreamID:7364977027650215022 Namespace:default ID:99f1d090-d56a-4290-9604-dc9ecaf9b879 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-04 11:12:40.226647903 +0800 CST StartTs:449522528450510852 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035c50e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522528476725250} {CheckpointTs:449522530691842052 MinTableBarrierTs:449522530691842053 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522530691842052, checkpointTs: 449522530691842052, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba2012bf-c3e5-4635-b98d-b794c8050714 {"id":"ba2012bf-c3e5-4635-b98d-b794c8050714","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792366} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41969fd770 ba2012bf-c3e5-4635-b98d-b794c8050714 /tidb/cdc/default/default/changefeed/info/99f1d090-d56a-4290-9604-dc9ecaf9b879 {"upstream-id":7364977027650215022,"namespace":"default","changefeed-id":"99f1d090-d56a-4290-9604-dc9ecaf9b879","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T11:12:40.226647903+08:00","start-ts":449522528450510852,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522528476725250} /tidb/cdc/default/default/changefeed/status/99f1d090-d56a-4290-9604-dc9ecaf9b879 {"checkpoint-ts":449522530691842052,"min-table-barrier-ts":449522530691842053,"admin-job-type":0} /tidb/cdc/default/default/task/position/ba2012bf-c3e5-4635-b98d-b794c8050714/99f1d090-d56a-4290-9604-dc9ecaf9b879 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977027650215022 {"id":7364977027650215022,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/99f1d090-d56a-4290-9604-dc9ecaf9b879 {UpstreamID:7364977027650215022 Namespace:default ID:99f1d090-d56a-4290-9604-dc9ecaf9b879 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-04 11:12:40.226647903 +0800 CST StartTs:449522528450510852 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035c50e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522528476725250} {CheckpointTs:449522530691842052 MinTableBarrierTs:449522530691842053 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522530691842052, checkpointTs: 449522530691842052, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba2012bf-c3e5-4635-b98d-b794c8050714 {"id":"ba2012bf-c3e5-4635-b98d-b794c8050714","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792366} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41969fd770 ba2012bf-c3e5-4635-b98d-b794c8050714 /tidb/cdc/default/default/changefeed/info/99f1d090-d56a-4290-9604-dc9ecaf9b879 {"upstream-id":7364977027650215022,"namespace":"default","changefeed-id":"99f1d090-d56a-4290-9604-dc9ecaf9b879","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T11:12:40.226647903+08:00","start-ts":449522528450510852,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522528476725250} /tidb/cdc/default/default/changefeed/status/99f1d090-d56a-4290-9604-dc9ecaf9b879 {"checkpoint-ts":449522530691842052,"min-table-barrier-ts":449522530691842053,"admin-job-type":0} /tidb/cdc/default/default/task/position/ba2012bf-c3e5-4635-b98d-b794c8050714/99f1d090-d56a-4290-9604-dc9ecaf9b879 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977027650215022 {"id":7364977027650215022,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/99f1d090-d56a-4290-9604-dc9ecaf9b879 {UpstreamID:7364977027650215022 Namespace:default ID:99f1d090-d56a-4290-9604-dc9ecaf9b879 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-04 11:12:40.226647903 +0800 CST StartTs:449522528450510852 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035c50e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522528476725250} {CheckpointTs:449522530691842052 MinTableBarrierTs:449522530691842053 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522530691842052, checkpointTs: 449522530691842052, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba2012bf-c3e5-4635-b98d-b794c8050714 {"id":"ba2012bf-c3e5-4635-b98d-b794c8050714","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792366} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41969fd770 ba2012bf-c3e5-4635-b98d-b794c8050714 /tidb/cdc/default/default/changefeed/info/99f1d090-d56a-4290-9604-dc9ecaf9b879 {"upstream-id":7364977027650215022,"namespace":"default","changefeed-id":"99f1d090-d56a-4290-9604-dc9ecaf9b879","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T11:12:40.226647903+08:00","start-ts":449522528450510852,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522528476725250} /tidb/cdc/default/default/changefeed/status/99f1d090-d56a-4290-9604-dc9ecaf9b879 {"checkpoint-ts":449522530691842052,"min-table-barrier-ts":449522530691842053,"admin-job-type":0} /tidb/cdc/default/default/task/position/ba2012bf-c3e5-4635-b98d-b794c8050714/99f1d090-d56a-4290-9604-dc9ecaf9b879 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977027650215022 + grep -q 'etcd info' {"id":7364977027650215022,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: mysql... <<================= "id": "ba2012bf-c3e5-4635-b98d-b794c8050714", "cluster-id": "default" run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:50 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81c5f373-435b-4b3c-a140-4dc2317b4856 {"id":"81c5f373-435b-4b3c-a140-4dc2317b4856","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792368} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196ca99cb 81c5f373-435b-4b3c-a140-4dc2317b4856 /tidb/cdc/default/default/upstream/7364977081367071706 {"id":7364977081367071706,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81c5f373-435b-4b3c-a140-4dc2317b4856 {"id":"81c5f373-435b-4b3c-a140-4dc2317b4856","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792368} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196ca99cb 81c5f373-435b-4b3c-a140-4dc2317b4856 /tidb/cdc/default/default/upstream/7364977081367071706 {"id":7364977081367071706,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81c5f373-435b-4b3c-a140-4dc2317b4856 {"id":"81c5f373-435b-4b3c-a140-4dc2317b4856","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792368} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196ca99cb 81c5f373-435b-4b3c-a140-4dc2317b4856 /tidb/cdc/default/default/upstream/7364977081367071706 {"id":7364977081367071706,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065c4ee40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242, pid:3739, start at 2024-05-04 11:12:51.178916243 +0800 CST m=+5.560935565 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:51.186 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:51.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:51.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) capture_id: ba2012bf-c3e5-4635-b98d-b794c8050714 check_processor_table_count http://127.0.0.1:2379 99f1d090-d56a-4290-9604-dc9ecaf9b879 ba2012bf-c3e5-4635-b98d-b794c8050714 1 run task successfully check_processor_table_count http://127.0.0.1:2379 99f1d090-d56a-4290-9604-dc9ecaf9b879 ba2012bf-c3e5-4635-b98d-b794c8050714 0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table count 1 does equal to expected count 0 run task failed 1-th time, retry later table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065c4ee40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242, pid:3739, start at 2024-05-04 11:12:51.178916243 +0800 CST m=+5.560935565 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:51.186 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:51.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:51.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065c50780017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242, pid:3824, start at 2024-05-04 11:12:51.270149766 +0800 CST m=+5.597430588 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:51.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:51.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:51.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later check_processor_table_count http://127.0.0.1:2379 99f1d090-d56a-4290-9604-dc9ecaf9b879 ba2012bf-c3e5-4635-b98d-b794c8050714 0 run task successfully ***************** properties ***************** "readproportion"="0" "scanproportion"="0" "updateproportion"="0" "threadcount"="4" "insertproportion"="0" "requestdistribution"="uniform" "workload"="core" "mysql.db"="changefeed_reconstruct" "recordcount"="50" "operationcount"="0" "dotransactions"="false" "readallfields"="true" "mysql.user"="root" "mysql.port"="4000" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 19.130189ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3157.2, Avg(us): 1481, Min(us): 989, Max(us): 4409, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later [Sat May 4 11:12:55 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.52045206.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_pause_resume.t3 exists table changefeed_reconstruct.usertable exists check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065cacb00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:3916, start at 2024-05-04 11:12:57.175856846 +0800 CST m=+5.350614184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:57.182 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:57.182 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:57.182 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:58 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60e77223-7500-45d1-88bb-c12fcf413b6b {"id":"60e77223-7500-45d1-88bb-c12fcf413b6b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792376} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196e856cc 60e77223-7500-45d1-88bb-c12fcf413b6b /tidb/cdc/default/default/upstream/7364977115887727135 {"id":7364977115887727135,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60e77223-7500-45d1-88bb-c12fcf413b6b {"id":"60e77223-7500-45d1-88bb-c12fcf413b6b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792376} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196e856cc 60e77223-7500-45d1-88bb-c12fcf413b6b /tidb/cdc/default/default/upstream/7364977115887727135 {"id":7364977115887727135,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60e77223-7500-45d1-88bb-c12fcf413b6b {"id":"60e77223-7500-45d1-88bb-c12fcf413b6b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792376} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196e856cc 60e77223-7500-45d1-88bb-c12fcf413b6b /tidb/cdc/default/default/upstream/7364977115887727135 {"id":7364977115887727135,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5264.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-manager wait process cdc.test exit for 3-th time... Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7364977115887727135,"namespace":"default","id":"ddl-manager","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-04T11:12:59.438984236+08:00","start_ts":449522533486821380,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522533486821380,"checkpoint_ts":449522533486821380,"checkpoint_time":"2024-05-04 11:12:59.329"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:12:59 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065cacb00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:3916, start at 2024-05-04 11:12:57.175856846 +0800 CST m=+5.350614184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:57.182 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:57.182 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:57.182 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065caf000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:3991, start at 2024-05-04 11:12:57.306173694 +0800 CST m=+5.425842514 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:57.314 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:57.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:57.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065cd5d00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt, pid:3746, start at 2024-05-04 11:12:59.767799064 +0800 CST m=+5.205180675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:59.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:59.764 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:59.764 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065cd5d00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt, pid:3746, start at 2024-05-04 11:12:59.767799064 +0800 CST m=+5.205180675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:59.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:59.764 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:59.764 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065cd8e40009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt, pid:3820, start at 2024-05-04 11:12:59.97187138 +0800 CST m=+5.337158994 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:59.980 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:59.961 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:59.961 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5243.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5186.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 2-th time, retry later + set +x + tso='449522534313623553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522534313623553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:13:03 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.52765278.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='449522534504988673 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522534504988673 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x wait process 5209 exit for 1-th time... check diff failed 3-th time, retry later wait process 5209 exit for 2-th time... wait process 5209 exit for 3-th time... [Sat May 4 11:13:05 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.52415243.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5209) - No such process wait process 5209 exit for 4-th time... process 5209 already exit [Sat May 4 11:13:06 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.53365338.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 20.21 secs (184548087 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:07 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af68cbc9-a52d-4ac6-ba51-1d92b70e197f {"id":"af68cbc9-a52d-4ac6-ba51-1d92b70e197f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792384} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197047ad5 af68cbc9-a52d-4ac6-ba51-1d92b70e197f /tidb/cdc/default/default/upstream/7364977141774175221 {"id":7364977141774175221,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af68cbc9-a52d-4ac6-ba51-1d92b70e197f {"id":"af68cbc9-a52d-4ac6-ba51-1d92b70e197f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792384} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197047ad5 af68cbc9-a52d-4ac6-ba51-1d92b70e197f /tidb/cdc/default/default/upstream/7364977141774175221 {"id":7364977141774175221,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af68cbc9-a52d-4ac6-ba51-1d92b70e197f {"id":"af68cbc9-a52d-4ac6-ba51-1d92b70e197f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792384} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197047ad5 af68cbc9-a52d-4ac6-ba51-1d92b70e197f /tidb/cdc/default/default/upstream/7364977141774175221 {"id":7364977141774175221,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5332.out cli changefeed create --start-ts=449522534313623553 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: cab78908-c46c-4cc8-825d-e9247978c1ea Info: {"upstream_id":7364977141774175221,"namespace":"default","id":"cab78908-c46c-4cc8-825d-e9247978c1ea","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:13:07.541671888+08:00","start_ts":449522534313623553,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522534313623553,"checkpoint_ts":449522534313623553,"checkpoint_time":"2024-05-04 11:13:02.483"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 4-th time, retry later + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:08 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c6ae697-d955-4f1a-857a-56e130a703aa {"id":"7c6ae697-d955-4f1a-857a-56e130a703aa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41970949f1 7c6ae697-d955-4f1a-857a-56e130a703aa /tidb/cdc/default/default/upstream/7364977154759645291 {"id":7364977154759645291,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c6ae697-d955-4f1a-857a-56e130a703aa {"id":"7c6ae697-d955-4f1a-857a-56e130a703aa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41970949f1 7c6ae697-d955-4f1a-857a-56e130a703aa /tidb/cdc/default/default/upstream/7364977154759645291 {"id":7364977154759645291,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c6ae697-d955-4f1a-857a-56e130a703aa {"id":"7c6ae697-d955-4f1a-857a-56e130a703aa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41970949f1 7c6ae697-d955-4f1a-857a-56e130a703aa /tidb/cdc/default/default/upstream/7364977154759645291 {"id":7364977154759645291,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5301.out cli changefeed create --start-ts=449522534504988673 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 2e68674f-b791-421e-8644-65b81df43951 Info: {"upstream_id":7364977154759645291,"namespace":"default","id":"2e68674f-b791-421e-8644-65b81df43951","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:13:09.275720073+08:00","start_ts":449522534504988673,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522534504988673,"checkpoint_ts":449522534504988673,"checkpoint_time":"2024-05-04 11:13:03.213"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:13:09 CST 2024] <<<<<< run test case sink_retry success! >>>>>> check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table drop_tables.c not exists for 1-th check, retry later + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table drop_tables.c not exists for 2-th check, retry later < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364977115887727135 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 11:12:59.438984236 +0800 CST StartTs:449522533486821380 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00319d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522533513035778} {CheckpointTs:449522533919358995 MinTableBarrierTs:449522533919358995 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/490c09a1-3eb5-4fdd-9017-e96f85abd12e {"id":"490c09a1-3eb5-4fdd-9017-e96f85abd12e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196e85952 490c09a1-3eb5-4fdd-9017-e96f85abd12e /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364977115887727135,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T11:12:59.438984236+08:00","start-ts":449522533486821380,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522533513035778} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522533919358995,"min-table-barrier-ts":449522533919358995,"admin-job-type":0} /tidb/cdc/default/default/task/position/490c09a1-3eb5-4fdd-9017-e96f85abd12e/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977115887727135 {"id":7364977115887727135,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364977115887727135 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 11:12:59.438984236 +0800 CST StartTs:449522533486821380 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00319d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522533513035778} {CheckpointTs:449522533919358995 MinTableBarrierTs:449522533919358995 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/490c09a1-3eb5-4fdd-9017-e96f85abd12e {"id":"490c09a1-3eb5-4fdd-9017-e96f85abd12e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196e85952 490c09a1-3eb5-4fdd-9017-e96f85abd12e /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364977115887727135,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T11:12:59.438984236+08:00","start-ts":449522533486821380,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522533513035778} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522533919358995,"min-table-barrier-ts":449522533919358995,"admin-job-type":0} /tidb/cdc/default/default/task/position/490c09a1-3eb5-4fdd-9017-e96f85abd12e/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977115887727135 {"id":7364977115887727135,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364977115887727135 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 11:12:59.438984236 +0800 CST StartTs:449522533486821380 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00319d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522533513035778} {CheckpointTs:449522533919358995 MinTableBarrierTs:449522533919358995 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/490c09a1-3eb5-4fdd-9017-e96f85abd12e {"id":"490c09a1-3eb5-4fdd-9017-e96f85abd12e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196e85952 490c09a1-3eb5-4fdd-9017-e96f85abd12e /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364977115887727135,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T11:12:59.438984236+08:00","start-ts":449522533486821380,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522533513035778} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522533919358995,"min-table-barrier-ts":449522533919358995,"admin-job-type":0} /tidb/cdc/default/default/task/position/490c09a1-3eb5-4fdd-9017-e96f85abd12e/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977115887727135 {"id":7364977115887727135,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Sat May 4 11:13:12 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.53955397.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065d95880018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn, pid:4093, start at 2024-05-04 11:13:12.057012422 +0800 CST m=+5.631682467 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:12.066 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:12.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:12.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065d95880018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn, pid:4093, start at 2024-05-04 11:13:12.057012422 +0800 CST m=+5.631682467 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:12.066 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:12.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:12.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065d90c00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-nsgc3-qtc3r-rm5zn, pid:4184, start at 2024-05-04 11:13:11.767698752 +0800 CST m=+5.260658320 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:11.774 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:11.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:11.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table drop_tables.c not exists for 3-th check, retry later [Sat May 4 11:13:15 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.54995501.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:15 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364977115887727135 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 11:12:59.438984236 +0800 CST StartTs:449522533486821380 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00319d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522533513035778} {CheckpointTs:449522533919358995 MinTableBarrierTs:449522533919358995 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/490c09a1-3eb5-4fdd-9017-e96f85abd12e {"id":"490c09a1-3eb5-4fdd-9017-e96f85abd12e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196e85952 490c09a1-3eb5-4fdd-9017-e96f85abd12e /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364977115887727135,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T11:12:59.438984236+08:00","start-ts":449522533486821380,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522533513035778} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522533945573393,"min-table-barrier-ts":449522533945573393,"admin-job-type":0} /tidb/cdc/default/default/task/position/490c09a1-3eb5-4fdd-9017-e96f85abd12e/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977115887727135 {"id":7364977115887727135,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364977115887727135 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 11:12:59.438984236 +0800 CST StartTs:449522533486821380 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00319d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522533513035778} {CheckpointTs:449522533919358995 MinTableBarrierTs:449522533919358995 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/490c09a1-3eb5-4fdd-9017-e96f85abd12e {"id":"490c09a1-3eb5-4fdd-9017-e96f85abd12e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196e85952 490c09a1-3eb5-4fdd-9017-e96f85abd12e /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364977115887727135,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T11:12:59.438984236+08:00","start-ts":449522533486821380,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522533513035778} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522533945573393,"min-table-barrier-ts":449522533945573393,"admin-job-type":0} /tidb/cdc/default/default/task/position/490c09a1-3eb5-4fdd-9017-e96f85abd12e/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977115887727135 {"id":7364977115887727135,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364977115887727135 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 11:12:59.438984236 +0800 CST StartTs:449522533486821380 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00319d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522533513035778} {CheckpointTs:449522533919358995 MinTableBarrierTs:449522533919358995 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/490c09a1-3eb5-4fdd-9017-e96f85abd12e {"id":"490c09a1-3eb5-4fdd-9017-e96f85abd12e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196e85952 490c09a1-3eb5-4fdd-9017-e96f85abd12e /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364977115887727135,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T11:12:59.438984236+08:00","start-ts":449522533486821380,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522533513035778} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522533945573393,"min-table-barrier-ts":449522533945573393,"admin-job-type":0} /tidb/cdc/default/default/task/position/490c09a1-3eb5-4fdd-9017-e96f85abd12e/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977115887727135 {"id":7364977115887727135,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later table drop_tables.c not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:18 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39e128f8-ba07-4acf-b30f-a7cce7c07e94 {"id":"39e128f8-ba07-4acf-b30f-a7cce7c07e94","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792395} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41973d84c7 39e128f8-ba07-4acf-b30f-a7cce7c07e94 /tidb/cdc/default/default/upstream/7364977205638409529 {"id":7364977205638409529,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39e128f8-ba07-4acf-b30f-a7cce7c07e94 {"id":"39e128f8-ba07-4acf-b30f-a7cce7c07e94","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792395} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41973d84c7 39e128f8-ba07-4acf-b30f-a7cce7c07e94 /tidb/cdc/default/default/upstream/7364977205638409529 {"id":7364977205638409529,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' table drop_tables.c not exists for 5-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 9.28 secs (401982153 bytes/sec) [Pipeline] { + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39e128f8-ba07-4acf-b30f-a7cce7c07e94 {"id":"39e128f8-ba07-4acf-b30f-a7cce7c07e94","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792395} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41973d84c7 39e128f8-ba07-4acf-b30f-a7cce7c07e94 /tidb/cdc/default/default/upstream/7364977205638409529 {"id":7364977205638409529,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5554.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /tmp/tidb_cdc_test/cdc/pulsar_test.toml [Pipeline] cache Create changefeed successfully! ID: e39e9174-0c4b-4fb3-9068-5a8dc345dc6b Info: {"upstream_id":7364977205638409529,"namespace":"default","id":"e39e9174-0c4b-4fb3-9068-5a8dc345dc6b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:13:18.873306163+08:00","start_ts":449522538587095041,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522538587095041,"checkpoint_ts":449522538587095041,"checkpoint_time":"2024-05-04 11:13:18.785"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 3-th check, retry later table drop_tables.c exists check diff failed 3-th time, retry later + set +x go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 check diff successfully go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading golang.org/x/text v0.14.0 wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 3-th time... check diff failed 4-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:13:22 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/google/uuid v1.6.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading golang.org/x/tools v0.20.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/golang/snappy v0.0.4 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/kr/pretty v0.3.1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 table ddl_manager.finish_mark not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da check diff failed 5-th time, retry later go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table ddl_manager.finish_mark not exists for 6-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065e56400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:6925, start at 2024-05-04 11:13:24.403907633 +0800 CST m=+5.344389730 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:24.413 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:24.417 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:24.417 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065e56400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:6925, start at 2024-05-04 11:13:24.403907633 +0800 CST m=+5.344389730 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:24.413 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:24.417 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:24.417 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065e57ac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:7010, start at 2024-05-04 11:13:24.483424234 +0800 CST m=+5.352417685 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:24.492 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:24.459 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:24.459 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8256.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449522541453115393 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522541453115393 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_manager.finish_mark not exists for 8-th check, retry later check diff failed 2-th time, retry later ***************** properties ***************** "updateproportion"="0" "mysql.db"="multi_capture_1" "readallfields"="true" "requestdistribution"="uniform" "workload"="core" "threadcount"="2" "mysql.user"="root" "readproportion"="0" "recordcount"="10" "operationcount"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.port"="4000" "scanproportion"="0" "insertproportion"="0" ********************************************** Run finished, takes 10.24291ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1894.9, Avg(us): 1977, Min(us): 964, Max(us): 5386, 95th(us): 6000, 99th(us): 6000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "mysql.host"="127.0.0.1" "readproportion"="0" "mysql.db"="multi_capture_2" "dotransactions"="false" "insertproportion"="0" "scanproportion"="0" "mysql.user"="root" "requestdistribution"="uniform" "readallfields"="true" "workload"="core" "updateproportion"="0" "mysql.port"="4000" "threadcount"="2" "recordcount"="10" "operationcount"="0" ********************************************** Run finished, takes 10.798954ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1610.6, Avg(us): 2070, Min(us): 1205, Max(us): 4502, 95th(us): 5000, 99th(us): 5000 table ddl_manager.finish_mark not exists for 9-th check, retry later ***************** properties ***************** "mysql.user"="root" "readproportion"="0" "dotransactions"="false" "scanproportion"="0" "mysql.port"="4000" "mysql.db"="multi_capture_3" "operationcount"="0" "recordcount"="10" "readallfields"="true" "workload"="core" "requestdistribution"="uniform" "threadcount"="2" "mysql.host"="127.0.0.1" "updateproportion"="0" "insertproportion"="0" ********************************************** Run finished, takes 10.930088ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1564.4, Avg(us): 2096, Min(us): 1219, Max(us): 4432, 95th(us): 5000, 99th(us): 5000 check diff failed 3-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 14.37 secs (259679617 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "readallfields"="true" "threadcount"="2" "dotransactions"="false" "mysql.user"="root" "readproportion"="0" "updateproportion"="0" "recordcount"="10" "mysql.port"="4000" "mysql.db"="multi_capture_4" "scanproportion"="0" "workload"="core" "requestdistribution"="uniform" "operationcount"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" ********************************************** Run finished, takes 10.435354ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1798.5, Avg(us): 2025, Min(us): 1093, Max(us): 4808, 95th(us): 5000, 99th(us): 5000 [Sat May 4 11:13:35 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.83788380.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 10-th check, retry later ***************** properties ***************** "threadcount"="2" "recordcount"="10" "mysql.port"="4000" "mysql.db"="testSync" "mysql.user"="root" "workload"="core" "readproportion"="0" "updateproportion"="0" "dotransactions"="false" "insertproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "operationcount"="0" "requestdistribution"="uniform" "scanproportion"="0" ********************************************** Run finished, takes 4.293911ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3298.1, Avg(us): 728, Min(us): 409, Max(us): 1649, 95th(us): 2000, 99th(us): 2000 check diff failed 4-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065eed9c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:3837, start at 2024-05-04 11:13:34.089466057 +0800 CST m=+5.339832192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:34.097 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:34.055 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:34.055 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065eed9c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:3837, start at 2024-05-04 11:13:34.089466057 +0800 CST m=+5.339832192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:34.097 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:34.055 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:34.055 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065eed880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:3906, start at 2024-05-04 11:13:34.091279604 +0800 CST m=+5.265335013 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:34.099 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:34.100 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:34.100 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 11-th check, retry later check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:38 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f85f2cf2-f465-4dff-bbbf-ddf04ab09551 {"id":"f85f2cf2-f465-4dff-bbbf-ddf04ab09551","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792415} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197690452 f85f2cf2-f465-4dff-bbbf-ddf04ab09551 /tidb/cdc/default/default/upstream/7364977258944836200 {"id":7364977258944836200,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f85f2cf2-f465-4dff-bbbf-ddf04ab09551 {"id":"f85f2cf2-f465-4dff-bbbf-ddf04ab09551","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792415} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197690452 f85f2cf2-f465-4dff-bbbf-ddf04ab09551 /tidb/cdc/default/default/upstream/7364977258944836200 {"id":7364977258944836200,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f85f2cf2-f465-4dff-bbbf-ddf04ab09551 {"id":"f85f2cf2-f465-4dff-bbbf-ddf04ab09551","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792415} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197690452 f85f2cf2-f465-4dff-bbbf-ddf04ab09551 /tidb/cdc/default/default/upstream/7364977258944836200 {"id":7364977258944836200,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 11:13:38 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.84318433.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5284.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 12-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff successfully + set +x + tso='449522544136945667 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522544136945667 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:41 GMT < Content-Length: 1259 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73125df8-64e9-4d5d-b047-5bf506529f01 {"id":"73125df8-64e9-4d5d-b047-5bf506529f01","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792419} /tidb/cdc/default/__cdc_meta__/capture/f85f2cf2-f465-4dff-bbbf-ddf04ab09551 {"id":"f85f2cf2-f465-4dff-bbbf-ddf04ab09551","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792415} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197690452 f85f2cf2-f465-4dff-bbbf-ddf04ab09551 /tidb/cdc/default/__cdc_meta__/owner/22318f419769048a 73125df8-64e9-4d5d-b047-5bf506529f01 /tidb/cdc/default/default/upstream/7364977258944836200 {"id":7364977258944836200,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73125df8-64e9-4d5d-b047-5bf506529f01 {"id":"73125df8-64e9-4d5d-b047-5bf506529f01","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792419} /tidb/cdc/default/__cdc_meta__/capture/f85f2cf2-f465-4dff-bbbf-ddf04ab09551 {"id":"f85f2cf2-f465-4dff-bbbf-ddf04ab09551","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792415} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197690452 f85f2cf2-f465-4dff-bbbf-ddf04ab09551 /tidb/cdc/default/__cdc_meta__/owner/22318f419769048a 73125df8-64e9-4d5d-b047-5bf506529f01 /tidb/cdc/default/default/upstream/7364977258944836200 {"id":7364977258944836200,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73125df8-64e9-4d5d-b047-5bf506529f01 {"id":"73125df8-64e9-4d5d-b047-5bf506529f01","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792419} /tidb/cdc/default/__cdc_meta__/capture/f85f2cf2-f465-4dff-bbbf-ddf04ab09551 {"id":"f85f2cf2-f465-4dff-bbbf-ddf04ab09551","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792415} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197690452 f85f2cf2-f465-4dff-bbbf-ddf04ab09551 /tidb/cdc/default/__cdc_meta__/owner/22318f419769048a 73125df8-64e9-4d5d-b047-5bf506529f01 /tidb/cdc/default/default/upstream/7364977258944836200 {"id":7364977258944836200,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 11:13:41 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.84838485.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 13-th check, retry later ***************** properties ***************** "readallfields"="true" "requestdistribution"="uniform" "scanproportion"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" "threadcount"="4" "mysql.user"="root" "workload"="core" "recordcount"="20" "mysql.port"="4000" "dotransactions"="false" "readproportion"="0" "operationcount"="0" "updateproportion"="0" "mysql.db"="changefeed_error" ********************************************** Run finished, takes 10.919885ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3173.7, Avg(us): 1959, Min(us): 988, Max(us): 4611, 95th(us): 5000, 99th(us): 5000 [Sat May 4 11:13:42 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.53505352.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 14-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:44 GMT < Content-Length: 1732 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73125df8-64e9-4d5d-b047-5bf506529f01 {"id":"73125df8-64e9-4d5d-b047-5bf506529f01","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792419} /tidb/cdc/default/__cdc_meta__/capture/80ffd24f-13cf-4137-85ea-2bb4e3eda015 {"id":"80ffd24f-13cf-4137-85ea-2bb4e3eda015","address":"127.0.0.1:8303","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792422} /tidb/cdc/default/__cdc_meta__/capture/f85f2cf2-f465-4dff-bbbf-ddf04ab09551 {"id":"f85f2cf2-f465-4dff-bbbf-ddf04ab09551","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792415} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197690452 f85f2cf2-f465-4dff-bbbf-ddf04ab09551 /tidb/cdc/default/__cdc_meta__/owner/22318f419769048a 73125df8-64e9-4d5d-b047-5bf506529f01 /tidb/cdc/default/__cdc_meta__/owner/22318f41976904a8 80ffd24f-13cf-4137-85ea-2bb4e3eda015 /tidb/cdc/default/default/upstream/7364977258944836200 {"id":7364977258944836200,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73125df8-64e9-4d5d-b047-5bf506529f01 {"id":"73125df8-64e9-4d5d-b047-5bf506529f01","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792419} /tidb/cdc/default/__cdc_meta__/capture/80ffd24f-13cf-4137-85ea-2bb4e3eda015 {"id":"80ffd24f-13cf-4137-85ea-2bb4e3eda015","address":"127.0.0.1:8303","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792422} /tidb/cdc/default/__cdc_meta__/capture/f85f2cf2-f465-4dff-bbbf-ddf04ab09551 {"id":"f85f2cf2-f465-4dff-bbbf-ddf04ab09551","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792415} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197690452 f85f2cf2-f465-4dff-bbbf-ddf04ab09551 /tidb/cdc/default/__cdc_meta__/owner/22318f419769048a 73125df8-64e9-4d5d-b047-5bf506529f01 /tidb/cdc/default/__cdc_meta__/owner/22318f41976904a8 80ffd24f-13cf-4137-85ea-2bb4e3eda015 /tidb/cdc/default/default/upstream/7364977258944836200 {"id":7364977258944836200,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73125df8-64e9-4d5d-b047-5bf506529f01 {"id":"73125df8-64e9-4d5d-b047-5bf506529f01","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792419} /tidb/cdc/default/__cdc_meta__/capture/80ffd24f-13cf-4137-85ea-2bb4e3eda015 {"id":"80ffd24f-13cf-4137-85ea-2bb4e3eda015","address":"127.0.0.1:8303","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792422} /tidb/cdc/default/__cdc_meta__/capture/f85f2cf2-f465-4dff-bbbf-ddf04ab09551 {"id":"f85f2cf2-f465-4dff-bbbf-ddf04ab09551","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792415} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197690452 f85f2cf2-f465-4dff-bbbf-ddf04ab09551 /tidb/cdc/default/__cdc_meta__/owner/22318f419769048a 73125df8-64e9-4d5d-b047-5bf506529f01 /tidb/cdc/default/__cdc_meta__/owner/22318f41976904a8 80ffd24f-13cf-4137-85ea-2bb4e3eda015 /tidb/cdc/default/default/upstream/7364977258944836200 {"id":7364977258944836200,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8544.out cli changefeed create --start-ts=449522541453115393 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 7.46 secs (500211096 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 1-th time, retry later Create changefeed successfully! ID: 3a50b798-f89f-49fe-8167-859a88dd7873 Info: {"upstream_id":7364977258944836200,"namespace":"default","id":"3a50b798-f89f-49fe-8167-859a88dd7873","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:13:45.20043631+08:00","start_ts":449522541453115393,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522541453115393,"checkpoint_ts":449522541453115393,"checkpoint_time":"2024-05-04 11:13:29.718"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:45 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 {"id":"bfe57403-c779-45ae-845a-3e1daf3c89f0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792422} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c3f6 bfe57403-c779-45ae-845a-3e1daf3c89f0 /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 {"id":"bfe57403-c779-45ae-845a-3e1daf3c89f0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792422} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c3f6 bfe57403-c779-45ae-845a-3e1daf3c89f0 /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 {"id":"bfe57403-c779-45ae-845a-3e1daf3c89f0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792422} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c3f6 bfe57403-c779-45ae-845a-3e1daf3c89f0 /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5414.out cli changefeed create --start-ts=449522544136945667 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error table ddl_manager.finish_mark not exists for 15-th check, retry later check diff failed 2-th time, retry later Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7364977299151316671,"namespace":"default","id":"changefeed-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:13:46.537745244+08:00","start_ts":449522544136945667,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522544136945667,"checkpoint_ts":449522544136945667,"checkpoint_time":"2024-05-04 11:13:39.956"} PASS + set +x table multi_capture_1.usertable not exists for 1-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 16-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449522544136945667, "checkpoint_time": "2024-05-04 11:13:39.956", "error": { "time": "2024-05-04T11:13:46.647191275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449522544136945367 is earlier than or equal to GC safepoint at 449522544136945667" } }' + echo '{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449522544136945667, "checkpoint_time": "2024-05-04 11:13:39.956", "error": { "time": "2024-05-04T11:13:46.647191275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449522544136945367 is earlier than or equal to GC safepoint at 449522544136945667" } }' { "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449522544136945667, "checkpoint_time": "2024-05-04 11:13:39.956", "error": { "time": "2024-05-04T11:13:46.647191275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449522544136945367 is earlier than or equal to GC safepoint at 449522544136945667" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449522544136945667, '"checkpoint_time":' '"2024-05-04' '11:13:39.956",' '"error":' '{' '"time":' '"2024-05-04T11:13:46.647191275+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449522544136945367 is earlier than or equal to GC safepoint at '449522544136945667"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449522544136945667, '"checkpoint_time":' '"2024-05-04' '11:13:39.956",' '"error":' '{' '"time":' '"2024-05-04T11:13:46.647191275+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449522544136945367 is earlier than or equal to GC safepoint at '449522544136945667"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449522544136945367 is earlier than or equal to GC safepoint at 449522544136945667' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449522544136945367 is earlier than or equal to GC safepoint at 449522544136945667 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5489.out cli changefeed resume -c changefeed-error table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 17-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table changefeed_error.usertable not exists for 1-th check, retry later check diff failed 4-th time, retry later table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 18-th check, retry later check diff failed 5-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065ff7040014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:6377, start at 2024-05-04 11:13:51.080092235 +0800 CST m=+5.698450722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:51.087 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:51.091 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:51.091 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065ff7040014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:6377, start at 2024-05-04 11:13:51.080092235 +0800 CST m=+5.698450722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:51.087 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:51.091 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:51.091 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065ff5580005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:6461, start at 2024-05-04 11:13:50.93831772 +0800 CST m=+5.500199290 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:50.945 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:50.934 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:50.934 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table changefeed_error.usertable exists check diff failed 1-th time, retry later table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 19-th check, retry later [Sat May 4 11:13:54 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.77627764.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully ***************** properties ***************** "workload"="core" "mysql.db"="changefeed_error" "dotransactions"="false" "recordcount"="20" "readallfields"="true" "insertproportion"="0" "mysql.port"="4000" "threadcount"="4" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "readproportion"="0" "mysql.user"="root" "operationcount"="0" "updateproportion"="0" "scanproportion"="0" ********************************************** Run finished, takes 11.386054ms INSERT - Takes(s): 0.0, Count: 19, OPS: 6895.1, Avg(us): 1852, Min(us): 457, Max(us): 8514, 95th(us): 9000, 99th(us): 9000 check diff successfully {"id":"bfe57403-c779-45ae-845a-3e1daf3c89f0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792422} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 + [[ /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later table multi_capture_4.usertable exists check diff failed 1-th time, retry later check diff successfully table ddl_manager.finish_mark not exists for 20-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 + [[ /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 10.16 secs (367316277 bytes/sec) [Pipeline] { [Pipeline] cache check diff successfully ***************** properties ***************** "operationcount"="0" "readallfields"="true" "scanproportion"="0" "updateproportion"="0" "recordcount"="20" "readproportion"="0" "insertproportion"="0" "requestdistribution"="uniform" "threadcount"="2" "mysql.db"="multi_capture_1" "mysql.port"="4000" "dotransactions"="false" "mysql.user"="root" "workload"="core" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 10.31142ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2210.4, Avg(us): 964, Min(us): 426, Max(us): 1606, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "insertproportion"="0" "threadcount"="2" "updateproportion"="0" "mysql.user"="root" "mysql.port"="4000" "dotransactions"="false" "recordcount"="20" "mysql.db"="multi_capture_2" "operationcount"="0" "mysql.host"="127.0.0.1" "workload"="core" "scanproportion"="0" "readallfields"="true" "readproportion"="0" "requestdistribution"="uniform" ********************************************** Run finished, takes 11.673064ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1887.3, Avg(us): 1000, Min(us): 488, Max(us): 1564, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "requestdistribution"="uniform" "workload"="core" "mysql.user"="root" "scanproportion"="0" "mysql.port"="4000" "readproportion"="0" "readallfields"="true" "recordcount"="20" "updateproportion"="0" "threadcount"="2" "mysql.host"="127.0.0.1" "dotransactions"="false" "insertproportion"="0" "mysql.db"="multi_capture_3" "operationcount"="0" ********************************************** Run finished, takes 11.383705ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2024.2, Avg(us): 1077, Min(us): 520, Max(us): 1619, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "threadcount"="2" "readproportion"="0" "scanproportion"="0" "mysql.db"="multi_capture_4" "requestdistribution"="uniform" "operationcount"="0" "workload"="core" "updateproportion"="0" "insertproportion"="0" "readallfields"="true" "mysql.port"="4000" "recordcount"="20" "mysql.host"="127.0.0.1" "mysql.user"="root" "dotransactions"="false" ********************************************** Run finished, takes 12.504284ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1744.2, Avg(us): 1081, Min(us): 557, Max(us): 1748, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:57 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ed67fa8-a08d-496e-844e-5c5a57941a60 {"id":"6ed67fa8-a08d-496e-844e-5c5a57941a60","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792434} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197d3bdc7 6ed67fa8-a08d-496e-844e-5c5a57941a60 /tidb/cdc/default/default/upstream/7364977372189067770 {"id":7364977372189067770,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ed67fa8-a08d-496e-844e-5c5a57941a60 {"id":"6ed67fa8-a08d-496e-844e-5c5a57941a60","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792434} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197d3bdc7 6ed67fa8-a08d-496e-844e-5c5a57941a60 /tidb/cdc/default/default/upstream/7364977372189067770 {"id":7364977372189067770,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ed67fa8-a08d-496e-844e-5c5a57941a60 {"id":"6ed67fa8-a08d-496e-844e-5c5a57941a60","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792434} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197d3bdc7 6ed67fa8-a08d-496e-844e-5c5a57941a60 /tidb/cdc/default/default/upstream/7364977372189067770 {"id":7364977372189067770,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 21-th check, retry later table processor_stop_delay.t not exists for 1-th check, retry later check diff successfully check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 22-th check, retry later wait process cdc.test exit for 1-th time... table processor_stop_delay.t exists check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later wait process cdc.test exit for 3-th time... check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 + [[ /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/bfe57403-c779-45ae-845a-3e1daf3c89f0 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:14:01 CST 2024] <<<<<< run test case multi_capture success! >>>>>> check diff successfully table ddl_manager.finish_mark not exists for 23-th check, retry later check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 24-th check, retry later check diff failed 1-th time, retry later check diff failed 4-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 6.42 secs (581432397 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 25-th check, retry later check diff failed 2-th time, retry later check diff failed 5-th time, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 11:14:06 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.57445746.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 26-th check, retry later check diff failed 3-th time, retry later check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:14:09 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/edc8fd73-4a6e-4c41-bfa5-abf02641be44 {"id":"edc8fd73-4a6e-4c41-bfa5-abf02641be44","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792447} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c4ed edc8fd73-4a6e-4c41-bfa5-abf02641be44 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7364977299151316671,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T11:13:46.537745244+08:00","start-ts":449522544136945667,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T11:14:07.392370368+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522551319691269} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449522547780747266,"min-table-barrier-ts":449522547780747268,"admin-job-type":1} /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/edc8fd73-4a6e-4c41-bfa5-abf02641be44 {"id":"edc8fd73-4a6e-4c41-bfa5-abf02641be44","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792447} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c4ed edc8fd73-4a6e-4c41-bfa5-abf02641be44 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7364977299151316671,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T11:13:46.537745244+08:00","start-ts":449522544136945667,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T11:14:07.392370368+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522551319691269} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449522547780747266,"min-table-barrier-ts":449522547780747268,"admin-job-type":1} /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/edc8fd73-4a6e-4c41-bfa5-abf02641be44 {"id":"edc8fd73-4a6e-4c41-bfa5-abf02641be44","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792447} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c4ed edc8fd73-4a6e-4c41-bfa5-abf02641be44 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7364977299151316671,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T11:13:46.537745244+08:00","start-ts":449522544136945667,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T11:14:07.392370368+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522551319691269} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449522547780747266,"min-table-barrier-ts":449522547780747268,"admin-job-type":1} /tidb/cdc/default/default/upstream/7364977299151316671 + grep -q 'etcd info' {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449522547780747266, "checkpoint_time": "2024-05-04 11:13:53.856", "error": { "time": "2024-05-04T11:14:07.392370368+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449522547780747266, "checkpoint_time": "2024-05-04 11:13:53.856", "error": { "time": "2024-05-04T11:14:07.392370368+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449522547780747266, "checkpoint_time": "2024-05-04 11:13:53.856", "error": { "time": "2024-05-04T11:14:07.392370368+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449522547780747266, '"checkpoint_time":' '"2024-05-04' '11:13:53.856",' '"error":' '{' '"time":' '"2024-05-04T11:14:07.392370368+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449522547780747266, '"checkpoint_time":' '"2024-05-04' '11:13:53.856",' '"error":' '{' '"time":' '"2024-05-04T11:14:07.392370368+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message table ddl_manager.finish_mark not exists for 27-th check, retry later + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5846.out cli changefeed remove -c changefeed-error check diff failed 4-th time, retry later Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449522547780747266 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 28-th check, retry later + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully check diff failed 5-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 11:14:14 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.59435945.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 29-th check, retry later check diff failed 6-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 30-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:14:17 GMT < Content-Length: 877 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d89eb551-8dbd-4172-9b08-176c17e57184 {"id":"d89eb551-8dbd-4172-9b08-176c17e57184","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792454} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c53e d89eb551-8dbd-4172-9b08-176c17e57184 /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d89eb551-8dbd-4172-9b08-176c17e57184 {"id":"d89eb551-8dbd-4172-9b08-176c17e57184","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792454} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c53e d89eb551-8dbd-4172-9b08-176c17e57184 /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d89eb551-8dbd-4172-9b08-176c17e57184 {"id":"d89eb551-8dbd-4172-9b08-176c17e57184","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792454} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c53e d89eb551-8dbd-4172-9b08-176c17e57184 /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6000.out cli changefeed create --start-ts=449522544136945667 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-1 wait process cdc.test exit for 2-th time... check diff failed 3-th time, retry later Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7364977299151316671,"namespace":"default","id":"changefeed-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:14:17.585335392+08:00","start_ts":449522544136945667,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522544136945667,"checkpoint_ts":449522544136945667,"checkpoint_time":"2024-05-04 11:13:39.956"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 3-th time... table ddl_manager.finish_mark not exists for 31-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:14:18 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x check diff failed 4-th time, retry later check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 1851 0 --:--:-- --:--:-- --:--:-- 1860 + info='{"state":"warning","resolved_ts":449522544530161665,"checkpoint_ts":449522544530161665,"last_warning":{"time":"2024-05-04T11:14:19.113536946+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449522544530161665,"checkpoint_ts":449522544530161665,"last_warning":{"time":"2024-05-04T11:14:19.113536946+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449522544530161665,"checkpoint_ts":449522544530161665,"last_warning":{"time":"2024-05-04T11:14:19.113536946+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ jq -r .state ++ echo '{"state":"warning","resolved_ts":449522544530161665,"checkpoint_ts":449522544530161665,"last_warning":{"time":"2024-05-04T11:14:19.113536946+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":449522544530161665,"checkpoint_ts":449522544530161665,"last_warning":{"time":"2024-05-04T11:14:19.113536946+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .last_warning.message + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6050.out cli changefeed remove -c changefeed-error-1 table ddl_manager.finish_mark not exists for 32-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449522544530161665 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 33-th check, retry later + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 11:14:23 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.61206122.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully table ddl_manager.finish_mark not exists for 34-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 16.73 secs (222972647 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 35-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066216080013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:9844, start at 2024-05-04 11:14:25.817225165 +0800 CST m=+5.467181026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:25.823 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:25.794 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:25.794 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:14:26 GMT < Content-Length: 877 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca0e52bf-e9bd-4e4c-b976-c5d55a877e6c {"id":"ca0e52bf-e9bd-4e4c-b976-c5d55a877e6c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792464} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c5be ca0e52bf-e9bd-4e4c-b976-c5d55a877e6c /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca0e52bf-e9bd-4e4c-b976-c5d55a877e6c {"id":"ca0e52bf-e9bd-4e4c-b976-c5d55a877e6c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792464} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c5be ca0e52bf-e9bd-4e4c-b976-c5d55a877e6c /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca0e52bf-e9bd-4e4c-b976-c5d55a877e6c {"id":"ca0e52bf-e9bd-4e4c-b976-c5d55a877e6c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792464} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c5be ca0e52bf-e9bd-4e4c-b976-c5d55a877e6c /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6175.out cli changefeed create --start-ts=449522544136945667 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7364977299151316671,"namespace":"default","id":"changefeed-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:14:27.155899524+08:00","start_ts":449522544136945667,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522544136945667,"checkpoint_ts":449522544136945667,"checkpoint_time":"2024-05-04 11:13:39.956"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 36-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066216080013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:9844, start at 2024-05-04 11:14:25.817225165 +0800 CST m=+5.467181026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:25.823 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:25.794 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:25.794 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0662183c0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-btg0j-ng7bd-q6vlm, pid:9912, start at 2024-05-04 11:14:25.942085319 +0800 CST m=+5.499588411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:25.949 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:25.935 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:25.935 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + info='{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449522544136945667, "checkpoint_time": "2024-05-04 11:13:39.956", "error": { "time": "2024-05-04T11:14:27.236800875+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522544136945667 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449522544136945667, "checkpoint_time": "2024-05-04 11:13:39.956", "error": { "time": "2024-05-04T11:14:27.236800875+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522544136945667 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449522544136945667, "checkpoint_time": "2024-05-04 11:13:39.956", "error": { "time": "2024-05-04T11:14:27.236800875+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522544136945667 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449522544136945667, '"checkpoint_time":' '"2024-05-04' '11:13:39.956",' '"error":' '{' '"time":' '"2024-05-04T11:14:27.236800875+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522544136945667 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449522544136945667, '"checkpoint_time":' '"2024-05-04' '11:13:39.956",' '"error":' '{' '"time":' '"2024-05-04T11:14:27.236800875+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522544136945667 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522544136945667 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522544136945667 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6258.out cli changefeed remove -c changefeed-error-2 check diff failed 2-th time, retry later Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449522544136945667 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 37-th check, retry later + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11167.out cli tso query --pd=http://127.0.0.1:2379 + set +x wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 38-th check, retry later check diff failed 3-th time, retry later wait process cdc.test exit for 2-th time... + set +x + tso='449522557529620481 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522557529620481 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449522557529620481 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sat May 4 11:14:32 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1121011212.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ***************** properties ***************** "mysql.user"="root" "dotransactions"="false" "recordcount"="10" "readproportion"="0" "scanproportion"="0" "threadcount"="2" "workload"="core" "operationcount"="0" "requestdistribution"="uniform" "readallfields"="true" "mysql.port"="4000" "updateproportion"="0" "mysql.db"="testSync" "mysql.host"="127.0.0.1" "insertproportion"="0" ********************************************** Run finished, takes 4.195067ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3680.3, Avg(us): 749, Min(us): 462, Max(us): 1705, 95th(us): 2000, 99th(us): 2000 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:14:32 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.63076309.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 39-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:14:35 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4f917cb-24fb-43bc-9284-5320491dcccc {"id":"d4f917cb-24fb-43bc-9284-5320491dcccc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792472} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198585ed1 d4f917cb-24fb-43bc-9284-5320491dcccc /tidb/cdc/default/default/upstream/7364977524188278669 {"id":7364977524188278669,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4f917cb-24fb-43bc-9284-5320491dcccc {"id":"d4f917cb-24fb-43bc-9284-5320491dcccc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792472} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198585ed1 d4f917cb-24fb-43bc-9284-5320491dcccc /tidb/cdc/default/default/upstream/7364977524188278669 + grep -q 'failed to get info:' {"id":7364977524188278669,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4f917cb-24fb-43bc-9284-5320491dcccc {"id":"d4f917cb-24fb-43bc-9284-5320491dcccc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792472} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198585ed1 d4f917cb-24fb-43bc-9284-5320491dcccc /tidb/cdc/default/default/upstream/7364977524188278669 {"id":7364977524188278669,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449522557529620481 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11271.out cli changefeed create --start-ts=449522557529620481 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml check diff failed 5-th time, retry later Create changefeed successfully! ID: test-1 Info: {"upstream_id":7364977524188278669,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:14:36.045011937+08:00","start_ts":449522557529620481,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522557529620481,"checkpoint_ts":449522557529620481,"checkpoint_time":"2024-05-04 11:14:31.045"} PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:14:35 GMT < Content-Length: 877 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f911459e-b377-447b-8e4b-fee9554a19b9 {"id":"f911459e-b377-447b-8e4b-fee9554a19b9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792473} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c622 f911459e-b377-447b-8e4b-fee9554a19b9 /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f911459e-b377-447b-8e4b-fee9554a19b9 {"id":"f911459e-b377-447b-8e4b-fee9554a19b9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792473} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c622 f911459e-b377-447b-8e4b-fee9554a19b9 /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f911459e-b377-447b-8e4b-fee9554a19b9 {"id":"f911459e-b377-447b-8e4b-fee9554a19b9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792473} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f419792c622 f911459e-b377-447b-8e4b-fee9554a19b9 /tidb/cdc/default/default/upstream/7364977299151316671 {"id":7364977299151316671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6359.out cli changefeed create --start-ts=0 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-initialize-error start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... coverage: 2.5% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7364977299151316671,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:14:36.246445658+08:00","start_ts":449522558856331269,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522558856331269,"checkpoint_ts":449522558856331269,"checkpoint_time":"2024-05-04 11:14:36.106"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 40-th check, retry later + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 1619 0 --:--:-- --:--:-- --:--:-- 1625 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-04 11:14:31.045","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-04 11:14:37.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '11:14:31.045","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '11:14:37.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '11:14:31.045","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '11:14:37.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s check diff successfully + sink_checkpoint_ts='2024-05-04 11:14:31.045' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '11:14:31.045","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '11:14:37.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '11:14:31.045","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '11:14:37.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-04 11:14:37' + echo 'sink_checkpoint_ts is 2024-05-04' 11:14:31.045 sink_checkpoint_ts is 2024-05-04 11:14:31.045 ++ date -d '2024-05-04 11:14:31.045' +%s + checkpoint_timestamp=1714792471 ++ date -d '2024-05-04 11:14:37' +%s + current_timestamp=1714792477 + '[' 6 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later + info='{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449522558856331269, "checkpoint_time": "2024-05-04 11:14:36.106", "error": { "time": "2024-05-04T11:14:36.426111392+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449522558856331269, "checkpoint_time": "2024-05-04 11:14:36.106", "error": { "time": "2024-05-04T11:14:36.426111392+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449522558856331269, "checkpoint_time": "2024-05-04 11:14:36.106", "error": { "time": "2024-05-04T11:14:36.426111392+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449522558856331269, '"checkpoint_time":' '"2024-05-04' '11:14:36.106",' '"error":' '{' '"time":' '"2024-05-04T11:14:36.426111392+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 41-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc) 3730333696 bytes in 11.94 secs (312321552 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh Verifying downstream PD is started... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=ce72417b-4844-4bc1-b825-8afee6de3bfe BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-wfls9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-wfls9 pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=2f6a0bd8-b1f4-447e-9674-b4a738bbc333 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-7mp2h GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-7mp2h pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449522558856331269, "checkpoint_time": "2024-05-04 11:14:36.106", "error": { "time": "2024-05-04T11:14:36.426111392+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449522558856331269, "checkpoint_time": "2024-05-04 11:14:36.106", "error": { "time": "2024-05-04T11:14:36.426111392+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449522558856331269, "checkpoint_time": "2024-05-04 11:14:36.106", "error": { "time": "2024-05-04T11:14:36.426111392+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449522558856331269, '"checkpoint_time":' '"2024-05-04' '11:14:36.106",' '"error":' '{' '"time":' '"2024-05-04T11:14:36.426111392+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later table ddl_manager.finish_mark not exists for 42-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=e2d72379-43e0-4df4-84ab-dc32ff240be3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-z4wx0 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-z4wx0 pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.t1 exists + sleep 5 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=0a9bf998-dd4e-4129-be64-02454057d8b0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-cgbl9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n pingcap_tiflow_pull_cdc_integration_test_1728-cgbl9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=a887d0ff-cf9f-4691-b35b-82f719cd83b1 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-w435w GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-w435w pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G19 Run cases: changefeed_fast_fail batch_update_to_no_batch changefeed_resume_with_checkpoint_ts PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=3ddd2de6-bb53-410c-bea7-177b32a8c2a4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G19 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-zcsxg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-zcsxg pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_fast_fail/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G20 Run cases: tidb_mysql_test ddl_reentrant multi_cdc_cluster PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=a06cddd5-e507-4e49-9881-78a04bb5bac8 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G20 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-g278r GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-g278r pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G14 Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=a33ac117-dfa6-4eb5-8a03-7304e40fd189 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-mdrtm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-mdrtm pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 43-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G13 [Pipeline] sh Run cases: tiflash region_merge common_1 PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=bc340ba8-f52c-4407-bc89-362bdc96b92d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-kk7r7 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2 pingcap_tiflow_pull_cdc_integration_test_1728-kk7r7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G21 Run cases: bank kill_owner_with_ddl owner_remove_table_error PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=1d364b1f-0748-4b3d-9ae3-97177b26beca BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G21 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-12kdt GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1 pingcap_tiflow_pull_cdc_integration_test_1728-12kdt GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G18 Run cases: availability http_proxies sequence PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=dd04a4d8-a42f-4e09-a63d-4daf3e27f2e7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G18 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-2z0sp GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b pingcap_tiflow_pull_cdc_integration_test_1728-2z0sp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/availability/run.sh using Sink-Type: mysql... <<================= Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage [Pipeline] } process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff/output" source-instances = ["tidb"] target-instance = "mysql" target-check-tables = ["consistent_replicate_ddl.usertable*","consistent_replicate_ddl.check*"] [data-sources] [data-sources.tidb] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449522527887949842" [data-sources.mysql] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/04 11:14:42.832 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/04 11:14:42.833 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/04 11:14:42.833 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/04 11:14:42.833 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/04 11:14:42.834 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=449522527376769036] [resolvedTs=449522527887949842] [2024/05/04 11:14:42.835 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.561426ms] [2024/05/04 11:14:42.836 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=2.530587ms] [2024/05/04 11:14:42.836 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 11:14:42.836 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 11:14:42.836 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 11:14:42.842 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 11:14:42.845 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/04 11:14:42.846 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/04 11:14:42.846 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 11:14:42.846 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 11:14:42.846 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 11:14:42.850 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 11:14:42.852 +08:00] [INFO] [mysql_ddl_sink.go:107] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/04 11:14:42.852 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527376769032,\"CommitTs\":449522527376769036,\"Query\":\"DROP TABLE `consistent_replicate_ddl`.`usertable1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":108,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":4,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:42.852 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [commitTs=449522527376769036] [namespace=] [changefeed=] start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=851b9320-e828-4168-9d3d-124c46eb3e28 JENKINS_NODE_COOKIE=6e7f7a36-9bec-4c4a-805c-da67b0569ddb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1728 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786590407547686912","prowjobid":"851b9320-e828-4168-9d3d-124c46eb3e28","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1728 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686912 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1728/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1728-0p427 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1728-0p427 pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1728 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: mysql... <<================= [2024/05/04 11:14:43.054 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [duration=201.746337ms] [namespace=] [changefeed=] [2024/05/04 11:14:43.054 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527416090632,\"CommitTs\":449522527416090636,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":114,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:43.054 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [commitTs=449522527416090636] [namespace=] [changefeed=] [2024/05/04 11:14:43.194 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [duration=139.591748ms] [namespace=] [changefeed=] [2024/05/04 11:14:43.194 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527468519429,\"CommitTs\":449522527468519433,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1_1\",\"TableID\":114,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:43.395 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [commitTs=449522527468519433] [namespace=] [changefeed=] [2024/05/04 11:14:43.533 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [duration=137.82674ms] [namespace=] [changefeed=] [2024/05/04 11:14:43.533 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527494733849,\"CommitTs\":449522527494733853,\"Query\":\"CREATE DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":1,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] The 1 times to try to start tidb cluster... [2024/05/04 11:14:43.734 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="CREATE DATABASE consistent_replicate_ddl1"] [commitTs=449522527494733853] [namespace=] [changefeed=] [2024/05/04 11:14:43.810 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="CREATE DATABASE consistent_replicate_ddl1"] [duration=75.782035ms] [namespace=] [changefeed=] [2024/05/04 11:14:43.810 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527520948250,\"CommitTs\":449522527520948253,\"Query\":\"ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":26,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:43.810 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [commitTs=449522527520948253] [namespace=] [changefeed=] [2024/05/04 11:14:43.882 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=72.465198ms] [namespace=] [changefeed=] [2024/05/04 11:14:43.882 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527547162646,\"CommitTs\":449522527547162650,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:43.882 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [commitTs=449522527547162650] [namespace=] [changefeed=] check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 11:14:43.956 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=73.979608ms] [namespace=] [changefeed=] [2024/05/04 11:14:43.957 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527625805842,\"CommitTs\":449522527625805846,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":6,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:44.157 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [commitTs=449522527625805846] [namespace=] [changefeed=] start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522560638648324, "checkpoint_time": "2024-05-04 11:14:42.905", "error": null }' + echo '{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522560638648324, "checkpoint_time": "2024-05-04 11:14:42.905", "error": null }' { "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522560638648324, "checkpoint_time": "2024-05-04 11:14:42.905", "error": null } ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449522560638648324, '"checkpoint_time":' '"2024-05-04' '11:14:42.905",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449522560638648324, '"checkpoint_time":' '"2024-05-04' '11:14:42.905",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6540.out cli changefeed pause -c changefeed-initialize-error The 1 times to try to start tidb cluster... [2024/05/04 11:14:44.415 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [duration=257.84925ms] [namespace=] [changefeed=] [2024/05/04 11:14:44.415 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527704449030,\"CommitTs\":449522527704449034,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":5,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:44.415 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [commitTs=449522527704449034] [namespace=] [changefeed=] table ddl_manager.finish_mark not exists for 44-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS [2024/05/04 11:14:44.676 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [duration=260.641628ms] [namespace=] [changefeed=] [2024/05/04 11:14:44.676 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527756877833,\"CommitTs\":449522527756877836,\"Query\":\"DROP DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":2,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:44.676 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="DROP DATABASE consistent_replicate_ddl1"] [commitTs=449522527756877836] [namespace=] [changefeed=] [2024/05/04 11:14:44.867 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="DROP DATABASE consistent_replicate_ddl1"] [duration=191.528873ms] [namespace=] [changefeed=] [2024/05/04 11:14:44.868 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527783092254,\"CommitTs\":449522527783092258,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:44.868 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [commitTs=449522527783092258] [namespace=] [changefeed=] [2024/05/04 11:14:44.942 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [duration=74.1336ms] [namespace=] [changefeed=] [2024/05/04 11:14:44.942 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527822413835,\"CommitTs\":449522527822413838,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:44.942 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [commitTs=449522527822413838] [namespace=] [changefeed=] start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/05/04 11:14:45.078 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [duration=136.024325ms] [namespace=] [changefeed=] [2024/05/04 11:14:45.078 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527822413835,\"CommitTs\":449522527822413838,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2_1\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:45.078 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [commitTs=449522527822413838] [namespace=] [changefeed=] [2024/05/04 11:14:45.213 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [duration=134.984311ms] [namespace=] [changefeed=] [2024/05/04 11:14:45.213 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527848628257,\"CommitTs\":449522527848628261,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":12,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:14:45.213 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [commitTs=449522527848628261] [namespace=] [changefeed=] start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 11:14:45.286 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [duration=73.300768ms] [namespace=] [changefeed=] [2024/05/04 11:14:45.287 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522527887949837,\"CommitTs\":449522527887949842,\"Query\":\"CREATE table consistent_replicate_ddl.check1(id int primary key)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"check1\",\"TableID\":129,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2467 0 --:--:-- --:--:-- --:--:-- 2479 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-04 11:14:43.395","puller_resolved_ts":"2024-05-04 11:14:37.795","last_synced_ts":"2024-05-04 11:14:37.845","now_ts":"2024-05-04 11:14:44.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-04' '11:14:43.395","puller_resolved_ts":"2024-05-04' '11:14:37.795","last_synced_ts":"2024-05-04' '11:14:37.845","now_ts":"2024-05-04' '11:14:44.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-04' '11:14:43.395","puller_resolved_ts":"2024-05-04' '11:14:37.795","last_synced_ts":"2024-05-04' '11:14:37.845","now_ts":"2024-05-04' '11:14:44.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 [2024/05/04 11:14:45.488 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [commitTs=449522527887949842] [namespace=] [changefeed=] [2024/05/04 11:14:45.564 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [duration=76.479837ms] [namespace=] [changefeed=] [2024/05/04 11:14:45.564 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}] [checkpointTs=449522527887949842] [2024/05/04 11:14:45.564 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}] [checkpointTs=449522527887949842] [2024/05/04 11:14:45.665 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449522527887949842] [2024/05/04 11:14:45.665 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449522527887949842] [2024/05/04 11:14:45.765 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449522527887949842] [2024/05/04 11:14:45.765 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449522527887949842] [2024/05/04 11:14:45.865 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=101] [appliedDDLCount=14] [currentCheckpoint=449522527887949842] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/04 11:14:45.865 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] Apply redo log successfully table consistent_replicate_ddl.check1 exists check diff successfully [Sat May 4 11:14:46 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> count(*) 50 check diff failed 3-th time, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449522560900792322, "checkpoint_time": "2024-05-04 11:14:43.905", "error": { "time": "2024-05-04T11:14:36.426111392+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449522560900792322, "checkpoint_time": "2024-05-04 11:14:43.905", "error": { "time": "2024-05-04T11:14:36.426111392+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449522560900792322, "checkpoint_time": "2024-05-04 11:14:43.905", "error": { "time": "2024-05-04T11:14:36.426111392+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449522560900792322, '"checkpoint_time":' '"2024-05-04' '11:14:43.905",' '"error":' '{' '"time":' '"2024-05-04T11:14:36.426111392+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449522560900792322, '"checkpoint_time":' '"2024-05-04' '11:14:43.905",' '"error":' '{' '"time":' '"2024-05-04T11:14:36.426111392+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message table ddl_manager.finish_mark not exists for 45-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6626.out cli changefeed resume -c changefeed-initialize-error The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 46-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522560900792322, "checkpoint_time": "2024-05-04 11:14:43.905", "error": null }' + echo '{ "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522560900792322, "checkpoint_time": "2024-05-04 11:14:43.905", "error": null }' { "upstream_id": 7364977299151316671, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522560900792322, "checkpoint_time": "2024-05-04 11:14:43.905", "error": null } ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449522560900792322, '"checkpoint_time":' '"2024-05-04' '11:14:43.905",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364977299151316671, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449522560900792322, '"checkpoint_time":' '"2024-05-04' '11:14:43.905",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6716.out cli changefeed remove -c changefeed-initialize-error Starting Upstream TiDB... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449522560900792322 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 5-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06636b480008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:9335, start at 2024-05-04 11:14:47.647513991 +0800 CST m=+5.329254602 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:47.654 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:47.634 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:47.634 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06636b480008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:9335, start at 2024-05-04 11:14:47.647513991 +0800 CST m=+5.329254602 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:47.654 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:47.634 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:47.634 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06636cd40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-793vm-h88dj-5md9w, pid:9399, start at 2024-05-04 11:14:47.774785054 +0800 CST m=+5.405995520 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:47.781 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:47.784 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:47.784 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 47-th check, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Upstream TiDB... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff successfully + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10657.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:14:52 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 48-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x + tso='449522563280797697 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522563280797697 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:14:54 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1070410706.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 49-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663eac40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx, pid:1234, start at 2024-05-04 11:14:55.814375864 +0800 CST m=+5.471688237 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:55.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:55.793 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:55.793 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663d3980008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01, pid:1241, start at 2024-05-04 11:14:54.313436422 +0800 CST m=+5.303324762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:54.323 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:54.310 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:54.310 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663d3980008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01, pid:1241, start at 2024-05-04 11:14:54.313436422 +0800 CST m=+5.303324762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:54.323 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:54.310 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:54.310 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663d4600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01, pid:1314, start at 2024-05-04 11:14:54.393313565 +0800 CST m=+5.331847331 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:54.401 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:54.360 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:54.360 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/clustered_index/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/clustered_index/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663d2a40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6, pid:1189, start at 2024-05-04 11:14:54.294092649 +0800 CST m=+5.303350097 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:54.301 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:54.299 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:54.299 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663d2a40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6, pid:1189, start at 2024-05-04 11:14:54.294092649 +0800 CST m=+5.303350097 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:54.301 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:54.299 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:54.299 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663d3580015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6, pid:1268, start at 2024-05-04 11:14:54.342844582 +0800 CST m=+5.281836220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:54.349 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:54.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:54.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 50-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663d638000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff, pid:1297, start at 2024-05-04 11:14:54.490944519 +0800 CST m=+5.273367110 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:54.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:54.478 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:54.478 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663d638000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff, pid:1297, start at 2024-05-04 11:14:54.490944519 +0800 CST m=+5.273367110 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:54.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:54.478 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:54.478 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663d87c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff, pid:1371, start at 2024-05-04 11:14:54.645507016 +0800 CST m=+5.357307974 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:54.656 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:54.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:54.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC2AB0F6693F0C < X-Xss-Protection: 1; mode=block < Date: Sat, 04 May 2024 03:14:57 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:14:57 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9e223ff-a9f3-4056-89a5-562529247679 {"id":"e9e223ff-a9f3-4056-89a5-562529247679","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792494} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198ae14d7 e9e223ff-a9f3-4056-89a5-562529247679 /tidb/cdc/default/default/upstream/7364977613390670176 {"id":7364977613390670176,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9e223ff-a9f3-4056-89a5-562529247679 {"id":"e9e223ff-a9f3-4056-89a5-562529247679","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792494} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198ae14d7 e9e223ff-a9f3-4056-89a5-562529247679 /tidb/cdc/default/default/upstream/7364977613390670176 {"id":7364977613390670176,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e9e223ff-a9f3-4056-89a5-562529247679 {"id":"e9e223ff-a9f3-4056-89a5-562529247679","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792494} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198ae14d7 e9e223ff-a9f3-4056-89a5-562529247679 /tidb/cdc/default/default/upstream/7364977613390670176 {"id":7364977613390670176,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10759.out cli changefeed create --start-ts=449522563280797697 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: e2ae8ff4-0797-4d9d-a5fd-093ecfe4c304 Info: {"upstream_id":7364977613390670176,"namespace":"default","id":"e2ae8ff4-0797-4d9d-a5fd-093ecfe4c304","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:14:58.021866549+08:00","start_ts":449522563280797697,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522563280797697,"checkpoint_ts":449522563280797697,"checkpoint_time":"2024-05-04 11:14:52.984"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066407680013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2, pid:1285, start at 2024-05-04 11:14:57.657958316 +0800 CST m=+5.777055638 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:57.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:57.676 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:57.676 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066407680013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2, pid:1285, start at 2024-05-04 11:14:57.657958316 +0800 CST m=+5.777055638 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:57.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:57.676 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:57.676 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06640ba80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2, pid:1355, start at 2024-05-04 11:14:57.926346749 +0800 CST m=+5.980220006 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:57.935 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:57.898 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:57.898 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663e6d8000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg, pid:1296, start at 2024-05-04 11:14:55.551517148 +0800 CST m=+5.365755426 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:55.557 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:55.542 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:55.542 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663e6d8000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg, pid:1296, start at 2024-05-04 11:14:55.551517148 +0800 CST m=+5.365755426 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:55.557 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:55.542 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:55.542 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663e9ec0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg, pid:1368, start at 2024-05-04 11:14:55.743521606 +0800 CST m=+5.507839819 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:55.750 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:55.739 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:55.739 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663eac40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx, pid:1234, start at 2024-05-04 11:14:55.814375864 +0800 CST m=+5.471688237 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:55.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:55.793 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:55.793 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663eb7c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx, pid:1317, start at 2024-05-04 11:14:55.873418058 +0800 CST m=+5.447831921 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:55.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:55.839 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:55.839 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/error.log Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 51-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0664131c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n, pid:1251, start at 2024-05-04 11:14:58.420414954 +0800 CST m=+5.353305580 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:58.427 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:58.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:58.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 11:14:59 CST 2024] <<<<<< START cdc server in clustered_index case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.clustered_index.26512653.out server --log-file /tmp/tidb_cdc_test/clustered_index/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/clustered_index/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Sat May 4 11:14:59 CST 2024] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.27002702.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x [Sat May 4 11:14:59 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.26012603.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0664084c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1, pid:1291, start at 2024-05-04 11:14:57.729095922 +0800 CST m=+5.597817508 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:57.735 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:57.734 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:57.734 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0664084c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1, pid:1291, start at 2024-05-04 11:14:57.729095922 +0800 CST m=+5.597817508 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:57.735 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:57.734 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:57.734 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06640be4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-12kdt-g1k5m-gxph1, pid:1368, start at 2024-05-04 11:14:57.923791173 +0800 CST m=+5.699784309 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:57.933 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:57.913 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:57.913 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bank/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bank/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bank/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bank/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bank/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2642.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2590.out cli tso query --pd=http://127.0.0.1:2379 [Sat May 4 11:15:00 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.26262628.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 52-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0664131c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n, pid:1251, start at 2024-05-04 11:14:58.420414954 +0800 CST m=+5.353305580 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:58.427 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:58.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:58.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066415580005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-cgbl9-ffvgz-74d8n, pid:1329, start at 2024-05-04 11:14:58.522253383 +0800 CST m=+5.406102287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:58.529 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:58.518 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:58.518 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06643c940017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b, pid:1195, start at 2024-05-04 11:15:01.053315959 +0800 CST m=+5.180086599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:01.059 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:01.029 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:01.029 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06643c940017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b, pid:1195, start at 2024-05-04 11:15:01.053315959 +0800 CST m=+5.180086599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:01.059 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:01.029 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:01.029 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06643e240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-2z0sp-0g62h-xqt5b, pid:1267, start at 2024-05-04 11:15:01.166154333 +0800 CST m=+5.224789694 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:01.175 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:01.179 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:01.179 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/availability/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/availability/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/availability/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/availability/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/availability/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > + set +x + tso='449522565393416193 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522565393416193 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:15:02 CST 2024] <<<<<< START cdc server in changefeed_fast_fail case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.26892691.out server --log-file /tmp/tidb_cdc_test/changefeed_fast_fail/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_fast_fail/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:02 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f85bc242-0994-417d-8397-2cb1d50d9946 {"id":"f85bc242-0994-417d-8397-2cb1d50d9946","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198c84dcb f85bc242-0994-417d-8397-2cb1d50d9946 /tidb/cdc/default/default/upstream/7364977642591842498 {"id":7364977642591842498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f85bc242-0994-417d-8397-2cb1d50d9946 {"id":"f85bc242-0994-417d-8397-2cb1d50d9946","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198c84dcb f85bc242-0994-417d-8397-2cb1d50d9946 /tidb/cdc/default/default/upstream/7364977642591842498 {"id":7364977642591842498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f85bc242-0994-417d-8397-2cb1d50d9946 {"id":"f85bc242-0994-417d-8397-2cb1d50d9946","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198c84dcb f85bc242-0994-417d-8397-2cb1d50d9946 /tidb/cdc/default/default/upstream/7364977642591842498 {"id":7364977642591842498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 65f2a569-528a-4323-851f-39211a4f7ded Info: {"upstream_id":7364977642591842498,"namespace":"default","id":"65f2a569-528a-4323-851f-39211a4f7ded","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-04T11:15:02.721067004+08:00","start_ts":449522564962451457,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522564962451457,"checkpoint_ts":449522564962451457,"checkpoint_time":"2024-05-04 11:14:59.399"} < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:02 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8dce347-4866-4a0b-afd6-f0eeaacc3e3e {"id":"a8dce347-4866-4a0b-afd6-f0eeaacc3e3e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198cd1dcf a8dce347-4866-4a0b-afd6-f0eeaacc3e3e /tidb/cdc/default/default/upstream/7364977650973973656 {"id":7364977650973973656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8dce347-4866-4a0b-afd6-f0eeaacc3e3e {"id":"a8dce347-4866-4a0b-afd6-f0eeaacc3e3e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198cd1dcf a8dce347-4866-4a0b-afd6-f0eeaacc3e3e /tidb/cdc/default/default/upstream/7364977650973973656 {"id":7364977650973973656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8dce347-4866-4a0b-afd6-f0eeaacc3e3e {"id":"a8dce347-4866-4a0b-afd6-f0eeaacc3e3e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198cd1dcf a8dce347-4866-4a0b-afd6-f0eeaacc3e3e /tidb/cdc/default/default/upstream/7364977650973973656 {"id":7364977650973973656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.2697.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 + set +x + tso='449522565349638145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522565349638145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:15:02 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.26322634.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 53-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:02 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0521ac92-8762-44e6-9f67-78812c033fae {"id":"0521ac92-8762-44e6-9f67-78812c033fae","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198ca4dca 0521ac92-8762-44e6-9f67-78812c033fae /tidb/cdc/default/default/upstream/7364977642894368614 {"id":7364977642894368614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0521ac92-8762-44e6-9f67-78812c033fae {"id":"0521ac92-8762-44e6-9f67-78812c033fae","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198ca4dca 0521ac92-8762-44e6-9f67-78812c033fae /tidb/cdc/default/default/upstream/7364977642894368614 {"id":7364977642894368614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0521ac92-8762-44e6-9f67-78812c033fae {"id":"0521ac92-8762-44e6-9f67-78812c033fae","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198ca4dca 0521ac92-8762-44e6-9f67-78812c033fae /tidb/cdc/default/default/upstream/7364977642894368614 {"id":7364977642894368614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: bca11640-5ba5-4f7a-baf3-2694f806894f Info: {"upstream_id":7364977642894368614,"namespace":"default","id":"bca11640-5ba5-4f7a-baf3-2694f806894f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:15:02.771393091+08:00","start_ts":449522564978442241,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522564978442241,"checkpoint_ts":449522564978442241,"checkpoint_time":"2024-05-04 11:14:59.460"} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 198a2155-09c2-43d5-9dc2-0706da48149f Info: {"upstream_id":7364977650973973656,"namespace":"default","id":"198a2155-09c2-43d5-9dc2-0706da48149f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:15:03.223494818+08:00","start_ts":449522565927141381,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522565927141381,"checkpoint_ts":449522565927141381,"checkpoint_time":"2024-05-04 11:15:03.079"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:03 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1facd0aa-d4c9-4509-bc73-c0e2e44ab3ed {"id":"1facd0aa-d4c9-4509-bc73-c0e2e44ab3ed","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792500} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d0dfcc 1facd0aa-d4c9-4509-bc73-c0e2e44ab3ed /tidb/cdc/default/default/upstream/7364977653118575242 {"id":7364977653118575242,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1facd0aa-d4c9-4509-bc73-c0e2e44ab3ed {"id":"1facd0aa-d4c9-4509-bc73-c0e2e44ab3ed","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792500} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d0dfcc 1facd0aa-d4c9-4509-bc73-c0e2e44ab3ed /tidb/cdc/default/default/upstream/7364977653118575242 {"id":7364977653118575242,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1facd0aa-d4c9-4509-bc73-c0e2e44ab3ed {"id":"1facd0aa-d4c9-4509-bc73-c0e2e44ab3ed","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792500} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d0dfcc 1facd0aa-d4c9-4509-bc73-c0e2e44ab3ed /tidb/cdc/default/default/upstream/7364977653118575242 {"id":7364977653118575242,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 11:15:03 CST 2024] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.26272629.out server --log-file /tmp/tidb_cdc_test/bank/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2677.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2624.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... check diff failed 5-th time, retry later + set +x Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449522566083641345 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522566083641345 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:15:05 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.27142716.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 54-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:05 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cab62a08-8fce-4415-8688-a598cea38c1b {"id":"cab62a08-8fce-4415-8688-a598cea38c1b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198cf81e4 cab62a08-8fce-4415-8688-a598cea38c1b /tidb/cdc/default/default/upstream/7364977652931164448 {"id":7364977652931164448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cab62a08-8fce-4415-8688-a598cea38c1b {"id":"cab62a08-8fce-4415-8688-a598cea38c1b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198cf81e4 cab62a08-8fce-4415-8688-a598cea38c1b /tidb/cdc/default/default/upstream/7364977652931164448 {"id":7364977652931164448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cab62a08-8fce-4415-8688-a598cea38c1b {"id":"cab62a08-8fce-4415-8688-a598cea38c1b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198cf81e4 cab62a08-8fce-4415-8688-a598cea38c1b /tidb/cdc/default/default/upstream/7364977652931164448 {"id":7364977652931164448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2749.out cli changefeed create --start-ts=449522565393416193 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-fast-fail + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:05 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ecee902-85d4-482b-b388-0eac912ba36a {"id":"6ecee902-85d4-482b-b388-0eac912ba36a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d38bcf 6ecee902-85d4-482b-b388-0eac912ba36a /tidb/cdc/default/default/upstream/7364977656874570308 {"id":7364977656874570308,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ecee902-85d4-482b-b388-0eac912ba36a {"id":"6ecee902-85d4-482b-b388-0eac912ba36a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d38bcf 6ecee902-85d4-482b-b388-0eac912ba36a /tidb/cdc/default/default/upstream/7364977656874570308 {"id":7364977656874570308,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ecee902-85d4-482b-b388-0eac912ba36a {"id":"6ecee902-85d4-482b-b388-0eac912ba36a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d38bcf 6ecee902-85d4-482b-b388-0eac912ba36a /tidb/cdc/default/default/upstream/7364977656874570308 {"id":7364977656874570308,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: b6c09b62-3700-4af9-bfe9-dd00da55264f Info: {"upstream_id":7364977656874570308,"namespace":"default","id":"b6c09b62-3700-4af9-bfe9-dd00da55264f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:15:05.840982098+08:00","start_ts":449522565349638145,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522565349638145,"checkpoint_ts":449522565349638145,"checkpoint_time":"2024-05-04 11:15:00.876"} cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table owner_resign.t1 exists check diff failed 1-th time, retry later Create changefeed successfully! ID: changefeed-fast-fail Info: {"upstream_id":7364977652931164448,"namespace":"default","id":"changefeed-fast-fail","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:15:06.157246698+08:00","start_ts":449522565393416193,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522565393416193,"checkpoint_ts":449522565393416193,"checkpoint_time":"2024-05-04 11:15:01.043"} PASS + set +x + tso='449522566307250177 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522566307250177 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:06 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e99839f-4c9e-46c8-8550-f8c390f61bcc {"id":"7e99839f-4c9e-46c8-8550-f8c390f61bcc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792503} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d48bdc 7e99839f-4c9e-46c8-8550-f8c390f61bcc /tidb/cdc/default/default/upstream/7364977662714085912 {"id":7364977662714085912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e99839f-4c9e-46c8-8550-f8c390f61bcc {"id":"7e99839f-4c9e-46c8-8550-f8c390f61bcc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792503} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d48bdc 7e99839f-4c9e-46c8-8550-f8c390f61bcc /tidb/cdc/default/default/upstream/7364977662714085912 {"id":7364977662714085912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e99839f-4c9e-46c8-8550-f8c390f61bcc {"id":"7e99839f-4c9e-46c8-8550-f8c390f61bcc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792503} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d48bdc 7e99839f-4c9e-46c8-8550-f8c390f61bcc /tidb/cdc/default/default/upstream/7364977662714085912 {"id":7364977662714085912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.2683.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] // container Create changefeed successfully! ID: f50fb4f6-c181-4343-812b-c395d5e02e25 Info: {"upstream_id":7364977662714085912,"namespace":"default","id":"f50fb4f6-c181-4343-812b-c395d5e02e25","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-04T11:15:06.762864348+08:00","start_ts":449522566859063298,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522566859063298,"checkpoint_ts":449522566859063298,"checkpoint_time":"2024-05-04 11:15:06.634"} PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066468940012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz, pid:1367, start at 2024-05-04 11:15:03.863452913 +0800 CST m=+5.325883666 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:03.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:03.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:03.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066468940012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz, pid:1367, start at 2024-05-04 11:15:03.863452913 +0800 CST m=+5.325883666 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:03.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:03.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:03.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06646938000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-z4wx0-w924w-x7hmz, pid:1438, start at 2024-05-04 11:15:03.901485945 +0800 CST m=+5.284619385 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:03.908 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:03.886 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:03.886 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06648a380016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v, pid:1364, start at 2024-05-04 11:15:06.028512122 +0800 CST m=+5.384592464 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:06.039 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:05.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:05.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06648ed40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z, pid:1251, start at 2024-05-04 11:15:06.318550635 +0800 CST m=+5.566953127 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:06.327 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:06.293 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:06.293 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... check diff failed 2-th time, retry later table cdc_tiflash_test.multi_data_type exists table ddl_manager.finish_mark not exists for 55-th check, retry later check diff failed 1-th time, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-fast-fail failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-fast-fail + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-fast-fail -s + info='{ "upstream_id": 7364977652931164448, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449522565393416193, "checkpoint_time": "2024-05-04 11:15:01.043", "error": { "time": "2024-05-04T11:15:06.169372948+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7364977652931164448, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449522565393416193, "checkpoint_time": "2024-05-04 11:15:01.043", "error": { "time": "2024-05-04T11:15:06.169372948+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7364977652931164448, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449522565393416193, "checkpoint_time": "2024-05-04 11:15:01.043", "error": { "time": "2024-05-04T11:15:06.169372948+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ echo '{' '"upstream_id":' 7364977652931164448, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 449522565393416193, '"checkpoint_time":' '"2024-05-04' '11:15:01.043",' '"error":' '{' '"time":' '"2024-05-04T11:15:06.169372948+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7364977652931164448, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 449522565393416193, '"checkpoint_time":' '"2024-05-04' '11:15:01.043",' '"error":' '{' '"time":' '"2024-05-04T11:15:06.169372948+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully run test case test_kill_owner [Sat May 4 11:15:07 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.26842686.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + GO111MODULE=on + go run bank.go case.go -u 'root@tcp(127.0.0.1:4000)/bank' -d 'root@tcp(127.0.0.1:3306)/bank' --test-round=20000 -a 127.0.0.1:20080 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.uber.org/multierr v1.11.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:08 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0662e956-00b6-42ff-9acc-004a36f29ff5 {"id":"0662e956-00b6-42ff-9acc-004a36f29ff5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792505} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d83bd7 0662e956-00b6-42ff-9acc-004a36f29ff5 /tidb/cdc/default/default/upstream/7364977663707653765 {"id":7364977663707653765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0662e956-00b6-42ff-9acc-004a36f29ff5 {"id":"0662e956-00b6-42ff-9acc-004a36f29ff5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792505} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d83bd7 0662e956-00b6-42ff-9acc-004a36f29ff5 /tidb/cdc/default/default/upstream/7364977663707653765 {"id":7364977663707653765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0662e956-00b6-42ff-9acc-004a36f29ff5 {"id":"0662e956-00b6-42ff-9acc-004a36f29ff5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792505} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198d83bd7 0662e956-00b6-42ff-9acc-004a36f29ff5 /tidb/cdc/default/default/upstream/7364977663707653765 {"id":7364977663707653765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2778.out cli changefeed create --start-ts=449522566083641345 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ check diff successfully [Sat May 4 11:15:08 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.28682870.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda Create changefeed successfully! ID: 404bd401-aa26-4ce4-a3c2-1eacde71ac40 Info: {"upstream_id":7364977663707653765,"namespace":"default","id":"404bd401-aa26-4ce4-a3c2-1eacde71ac40","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:15:08.720814766+08:00","start_ts":449522566083641345,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522566083641345,"checkpoint_ts":449522566083641345,"checkpoint_time":"2024-05-04 11:15:03.676"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06648a380016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v, pid:1364, start at 2024-05-04 11:15:06.028512122 +0800 CST m=+5.384592464 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:06.039 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:05.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:05.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06648bc40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-0p427-q4gbg-rgv3v, pid:1441, start at 2024-05-04 11:15:06.103816067 +0800 CST m=+5.378667813 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:06.111 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:06.097 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:06.097 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06648ed40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z, pid:1251, start at 2024-05-04 11:15:06.318550635 +0800 CST m=+5.566953127 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:06.327 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:06.293 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:06.293 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06648f8c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-g278r-3f91t-d6v8z, pid:1327, start at 2024-05-04 11:15:06.377259609 +0800 CST m=+5.543618334 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:06.388 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:06.389 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:06.389 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2703.out cli tso query --pd=http://127.0.0.1:2379 TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 56-th check, retry later wait process cdc.test exit for 2-th time... check diff successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:15:10 CST 2024] <<<<<< run test case changefeed_fast_fail success! >>>>>> check diff successfully wait process cdc.test exit for 1-th time... + set +x go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading golang.org/x/time v0.5.0 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/coreos/go-semver v0.3.1 wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x + tso='449522567569735681 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522567569735681 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:15:10 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.27442746.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading golang.org/x/text v0.14.0 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.cli.2684.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/198a2155-09c2-43d5-9dc2-0706da48149f {UpstreamID:7364977650973973656 Namespace:default ID:198a2155-09c2-43d5-9dc2-0706da48149f SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:03.223494818 +0800 CST StartTs:449522565927141381 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00249d0e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522565953093641} {CheckpointTs:449522567814578185 MinTableBarrierTs:449522567814578185 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8dce347-4866-4a0b-afd6-f0eeaacc3e3e {"id":"a8dce347-4866-4a0b-afd6-f0eeaacc3e3e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/capture/c52acc8f-3800-4b57-b620-588bed1093a3 {"id":"c52acc8f-3800-4b57-b620-588bed1093a3","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792508} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198cd1dcf a8dce347-4866-4a0b-afd6-f0eeaacc3e3e /tidb/cdc/default/__cdc_meta__/owner/22318f4198cd1e65 c52acc8f-3800-4b57-b620-588bed1093a3 /tidb/cdc/default/default/changefeed/info/198a2155-09c2-43d5-9dc2-0706da48149f {"upstream-id":7364977650973973656,"namespace":"default","changefeed-id":"198a2155-09c2-43d5-9dc2-0706da48149f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:03.223494818+08:00","start-ts":449522565927141381,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522565953093641} /tidb/cdc/default/default/changefeed/status/198a2155-09c2-43d5-9dc2-0706da48149f {"checkpoint-ts":449522567814578185,"min-table-barrier-ts":449522567814578185,"admin-job-type":0} /tidb/cdc/default/default/task/position/a8dce347-4866-4a0b-afd6-f0eeaacc3e3e/198a2155-09c2-43d5-9dc2-0706da48149f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c52acc8f-3800-4b57-b620-588bed1093a3/198a2155-09c2-43d5-9dc2-0706da48149f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977650973973656 {"id":7364977650973973656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/198a2155-09c2-43d5-9dc2-0706da48149f {UpstreamID:7364977650973973656 Namespace:default ID:198a2155-09c2-43d5-9dc2-0706da48149f SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:03.223494818 +0800 CST StartTs:449522565927141381 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00249d0e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522565953093641} {CheckpointTs:449522567814578185 MinTableBarrierTs:449522567814578185 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8dce347-4866-4a0b-afd6-f0eeaacc3e3e {"id":"a8dce347-4866-4a0b-afd6-f0eeaacc3e3e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/capture/c52acc8f-3800-4b57-b620-588bed1093a3 {"id":"c52acc8f-3800-4b57-b620-588bed1093a3","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792508} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198cd1dcf a8dce347-4866-4a0b-afd6-f0eeaacc3e3e /tidb/cdc/default/__cdc_meta__/owner/22318f4198cd1e65 c52acc8f-3800-4b57-b620-588bed1093a3 /tidb/cdc/default/default/changefeed/info/198a2155-09c2-43d5-9dc2-0706da48149f {"upstream-id":7364977650973973656,"namespace":"default","changefeed-id":"198a2155-09c2-43d5-9dc2-0706da48149f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:03.223494818+08:00","start-ts":449522565927141381,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522565953093641} /tidb/cdc/default/default/changefeed/status/198a2155-09c2-43d5-9dc2-0706da48149f {"checkpoint-ts":449522567814578185,"min-table-barrier-ts":449522567814578185,"admin-job-type":0} /tidb/cdc/default/default/task/position/a8dce347-4866-4a0b-afd6-f0eeaacc3e3e/198a2155-09c2-43d5-9dc2-0706da48149f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c52acc8f-3800-4b57-b620-588bed1093a3/198a2155-09c2-43d5-9dc2-0706da48149f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977650973973656 {"id":7364977650973973656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/198a2155-09c2-43d5-9dc2-0706da48149f {UpstreamID:7364977650973973656 Namespace:default ID:198a2155-09c2-43d5-9dc2-0706da48149f SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:03.223494818 +0800 CST StartTs:449522565927141381 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00249d0e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522565953093641} {CheckpointTs:449522567814578185 MinTableBarrierTs:449522567814578185 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a8dce347-4866-4a0b-afd6-f0eeaacc3e3e {"id":"a8dce347-4866-4a0b-afd6-f0eeaacc3e3e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/capture/c52acc8f-3800-4b57-b620-588bed1093a3 {"id":"c52acc8f-3800-4b57-b620-588bed1093a3","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792508} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198cd1dcf a8dce347-4866-4a0b-afd6-f0eeaacc3e3e /tidb/cdc/default/__cdc_meta__/owner/22318f4198cd1e65 c52acc8f-3800-4b57-b620-588bed1093a3 /tidb/cdc/default/default/changefeed/info/198a2155-09c2-43d5-9dc2-0706da48149f {"upstream-id":7364977650973973656,"namespace":"default","changefeed-id":"198a2155-09c2-43d5-9dc2-0706da48149f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:03.223494818+08:00","start-ts":449522565927141381,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522565953093641} /tidb/cdc/default/default/changefeed/status/198a2155-09c2-43d5-9dc2-0706da48149f {"checkpoint-ts":449522567814578185,"min-table-barrier-ts":449522567814578185,"admin-job-type":0} /tidb/cdc/default/default/task/position/a8dce347-4866-4a0b-afd6-f0eeaacc3e3e/198a2155-09c2-43d5-9dc2-0706da48149f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c52acc8f-3800-4b57-b620-588bed1093a3/198a2155-09c2-43d5-9dc2-0706da48149f + grep -q 'etcd info' {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977650973973656 {"id":7364977650973973656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 57-th check, retry later check diff failed 1-th time, retry later wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:10 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fbb79572-a0d0-4686-b6f7-cf905d9d0b12 {"id":"fbb79572-a0d0-4686-b6f7-cf905d9d0b12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792508} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32f02 fbb79572-a0d0-4686-b6f7-cf905d9d0b12 /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fbb79572-a0d0-4686-b6f7-cf905d9d0b12 {"id":"fbb79572-a0d0-4686-b6f7-cf905d9d0b12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792508} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32f02 fbb79572-a0d0-4686-b6f7-cf905d9d0b12 /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fbb79572-a0d0-4686-b6f7-cf905d9d0b12 {"id":"fbb79572-a0d0-4686-b6f7-cf905d9d0b12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792508} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32f02 fbb79572-a0d0-4686-b6f7-cf905d9d0b12 /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2742.out cli changefeed create --start-ts=449522566307250177 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 2264ab66-67c2-491a-8dc9-21a79802a34c Info: {"upstream_id":7364977674701950479,"namespace":"default","id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:15:11.267430246+08:00","start_ts":449522566307250177,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522566307250177,"checkpoint_ts":449522566307250177,"checkpoint_time":"2024-05-04 11:15:04.529"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2715.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:15:11 CST 2024] <<<<<< run test case tiflash success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + set +x + tso='449522568081965057 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522568081965057 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:15:12 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.27492751.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449522568119451650 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522568119451650 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:15:12 CST 2024] <<<<<< START cdc server in tidb_mysql_test case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.27252727.out server --log-file /tmp/tidb_cdc_test/tidb_mysql_test/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tidb_mysql_test/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/tidwall/btree v1.7.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/stretchr/testify v1.9.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/golang/snappy v0.0.4 table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 58-th check, retry later check diff failed 2-th time, retry later go: downloading github.com/dolthub/swiss v0.2.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:13 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1617e1d1-5db7-4db1-9edc-ca6c7a061453 {"id":"1617e1d1-5db7-4db1-9edc-ca6c7a061453","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792511} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198ed97d3 1617e1d1-5db7-4db1-9edc-ca6c7a061453 /tidb/cdc/default/default/upstream/7364977688407894805 {"id":7364977688407894805,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1617e1d1-5db7-4db1-9edc-ca6c7a061453 {"id":"1617e1d1-5db7-4db1-9edc-ca6c7a061453","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792511} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198ed97d3 1617e1d1-5db7-4db1-9edc-ca6c7a061453 /tidb/cdc/default/default/upstream/7364977688407894805 {"id":7364977688407894805,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1617e1d1-5db7-4db1-9edc-ca6c7a061453 {"id":"1617e1d1-5db7-4db1-9edc-ca6c7a061453","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792511} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198ed97d3 1617e1d1-5db7-4db1-9edc-ca6c7a061453 /tidb/cdc/default/default/upstream/7364977688407894805 {"id":7364977688407894805,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2801.out cli changefeed create --start-ts=449522567569735681 --sink-uri=mysql://normal:123456@127.0.0.1:3306/tidb-txn-mode=pessimistic Create changefeed successfully! ID: 91122b4e-2186-4c33-ab10-f41bd50de2db Info: {"upstream_id":7364977688407894805,"namespace":"default","id":"91122b4e-2186-4c33-ab10-f41bd50de2db","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/tidb-txn-mode=pessimistic","create_time":"2024-05-04T11:15:14.358669594+08:00","start_ts":449522567569735681,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522567569735681,"checkpoint_ts":449522567569735681,"checkpoint_time":"2024-05-04 11:15:09.345"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 "is-owner": true, run task successfully table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later check diff failed 3-th time, retry later check diff failed 1-th time, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 59-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:15 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39bb1160-9c21-4b42-88fa-398be28c178c {"id":"39bb1160-9c21-4b42-88fa-398be28c178c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792513} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198f7c4d4 39bb1160-9c21-4b42-88fa-398be28c178c /tidb/cdc/default/default/upstream/7364977696558169146 {"id":7364977696558169146,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39bb1160-9c21-4b42-88fa-398be28c178c {"id":"39bb1160-9c21-4b42-88fa-398be28c178c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792513} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198f7c4d4 39bb1160-9c21-4b42-88fa-398be28c178c /tidb/cdc/default/default/upstream/7364977696558169146 {"id":7364977696558169146,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39bb1160-9c21-4b42-88fa-398be28c178c {"id":"39bb1160-9c21-4b42-88fa-398be28c178c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792513} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198f7c4d4 39bb1160-9c21-4b42-88fa-398be28c178c /tidb/cdc/default/default/upstream/7364977696558169146 {"id":7364977696558169146,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2811.out cli changefeed create --start-ts=449522568081965057 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:16 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07e5da6c-14ba-4868-816b-d323be102ebe {"id":"07e5da6c-14ba-4868-816b-d323be102ebe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792513} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198f62cd3 07e5da6c-14ba-4868-816b-d323be102ebe /tidb/cdc/default/default/upstream/7364977695203027671 {"id":7364977695203027671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07e5da6c-14ba-4868-816b-d323be102ebe {"id":"07e5da6c-14ba-4868-816b-d323be102ebe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792513} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198f62cd3 07e5da6c-14ba-4868-816b-d323be102ebe /tidb/cdc/default/default/upstream/7364977695203027671 {"id":7364977695203027671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07e5da6c-14ba-4868-816b-d323be102ebe {"id":"07e5da6c-14ba-4868-816b-d323be102ebe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792513} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198f62cd3 07e5da6c-14ba-4868-816b-d323be102ebe /tidb/cdc/default/default/upstream/7364977695203027671 {"id":7364977695203027671,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 142 0 0 100 142 0 1498 --:--:-- --:--:-- --:--:-- 1510 mysql_test start go: downloading github.com/maxshuang/mysql-tester v0.0.2 + set +x go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/zap v1.27.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/google/btree v1.1.2 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 Create changefeed successfully! ID: 68a35e9a-3e60-4944-b44c-66ce69150647 Info: {"upstream_id":7364977696558169146,"namespace":"default","id":"68a35e9a-3e60-4944-b44c-66ce69150647","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:15:16.374335733+08:00","start_ts":449522568081965057,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522568081965057,"checkpoint_ts":449522568081965057,"checkpoint_time":"2024-05-04 11:15:11.299"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] // withEnv Verifying downstream PD is started... go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 4-th time, retry later go: downloading golang.org/x/text v0.14.0 owner pid: 2689 owner id fbb79572-a0d0-4686-b6f7-cf905d9d0b12 [Sat May 4 11:15:17 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.28582860.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.t2 not exists for 2-th check, retry later [Pipeline] } check diff failed 2-th time, retry later check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 60-th check, retry later [Pipeline] // node go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 + set +x go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 [Pipeline] } [Pipeline] // podTemplate [Pipeline] } go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/text v0.14.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20211122065349-4d16aabf900b go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/sirupsen/logrus v1.8.1 go: downloading github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3 go: downloading go.uber.org/atomic v1.9.0 go: downloading golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 go: downloading golang.org/x/text v0.3.7 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 go: downloading go.uber.org/zap v1.19.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 go: downloading go.uber.org/multierr v1.7.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0665204c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q, pid:4225, start at 2024-05-04 11:15:15.643359205 +0800 CST m=+5.308468865 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:15.650 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:15.653 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:15.653 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0665204c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q, pid:4225, start at 2024-05-04 11:15:15.643359205 +0800 CST m=+5.308468865 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:15.650 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:15.653 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:15.653 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066522900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q, pid:4302, start at 2024-05-04 11:15:15.772700703 +0800 CST m=+5.386248374 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:15.779 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:15.748 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:15.748 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading cloud.google.com/go/iam v1.1.7 table ddl_manager.finish_mark not exists for 61-th check, retry later check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 check diff failed 4-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 check diff failed 5-th time, retry later go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026a6750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522570082123784 MinTableBarrierTs:449522570344267790 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d39b3a72-5efe-493b-8087-59f5f3ec86a3 {"id":"d39b3a72-5efe-493b-8087-59f5f3ec86a3","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792517} /tidb/cdc/default/__cdc_meta__/capture/fbb79572-a0d0-4686-b6f7-cf905d9d0b12 {"id":"fbb79572-a0d0-4686-b6f7-cf905d9d0b12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792508} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32f02 fbb79572-a0d0-4686-b6f7-cf905d9d0b12 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32f70 d39b3a72-5efe-493b-8087-59f5f3ec86a3 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522570082123788,"min-table-barrier-ts":449522570344267790,"admin-job-type":0} /tidb/cdc/default/default/task/position/d39b3a72-5efe-493b-8087-59f5f3ec86a3/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fbb79572-a0d0-4686-b6f7-cf905d9d0b12/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026a6750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522570082123784 MinTableBarrierTs:449522570344267790 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d39b3a72-5efe-493b-8087-59f5f3ec86a3 {"id":"d39b3a72-5efe-493b-8087-59f5f3ec86a3","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792517} /tidb/cdc/default/__cdc_meta__/capture/fbb79572-a0d0-4686-b6f7-cf905d9d0b12 {"id":"fbb79572-a0d0-4686-b6f7-cf905d9d0b12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792508} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32f02 fbb79572-a0d0-4686-b6f7-cf905d9d0b12 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32f70 d39b3a72-5efe-493b-8087-59f5f3ec86a3 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522570082123788,"min-table-barrier-ts":449522570344267790,"admin-job-type":0} /tidb/cdc/default/default/task/position/d39b3a72-5efe-493b-8087-59f5f3ec86a3/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fbb79572-a0d0-4686-b6f7-cf905d9d0b12/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026a6750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522570082123784 MinTableBarrierTs:449522570344267790 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d39b3a72-5efe-493b-8087-59f5f3ec86a3 {"id":"d39b3a72-5efe-493b-8087-59f5f3ec86a3","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792517} /tidb/cdc/default/__cdc_meta__/capture/fbb79572-a0d0-4686-b6f7-cf905d9d0b12 {"id":"fbb79572-a0d0-4686-b6f7-cf905d9d0b12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792508} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32f02 fbb79572-a0d0-4686-b6f7-cf905d9d0b12 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32f70 d39b3a72-5efe-493b-8087-59f5f3ec86a3 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522570082123788,"min-table-barrier-ts":449522570344267790,"admin-job-type":0} /tidb/cdc/default/default/task/position/d39b3a72-5efe-493b-8087-59f5f3ec86a3/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fbb79572-a0d0-4686-b6f7-cf905d9d0b12/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "fbb79572-a0d0-4686-b6f7-cf905d9d0b12" | grep -v cluster_id | grep id go: downloading k8s.io/apimachinery v0.28.6 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading golang.org/x/sync v0.7.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b [Sat May 4 11:15:20 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.56805682.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:15:19.858 +08:00] [INFO] [case.go:445] ["open db success"] [dsn=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:15:19.859 +08:00] [INFO] [case.go:445] ["open db success"] [dsn=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:15:19.860 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq0] [2024/05/04 11:15:20.380 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.736737ms"] [query="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/04 11:15:20.380 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq1] go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading golang.org/x/time v0.5.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 table partition_table.finish_mark not exists for 2-th check, retry later [2024/05/04 11:15:20.895 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.389434ms"] [query="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/04 11:15:20.895 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq2] go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/dolthub/maphash v0.1.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/kylelemons/godebug v1.1.0 table clustered_index_test.t0 exists table clustered_index_test.t1 exists table clustered_index_test.t2 exists check table exists success check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 62-th check, retry later check diff failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 [2024/05/04 11:15:21.410 +08:00] [INFO] [case.go:365] ["insert 100 takes 5.036177ms"] [query="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/04 11:15:21.411 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq3] wait process cdc.test exit for 1-th time... go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 check diff failed 2-th time, retry later "cluster-id": "default" "id": "d39b3a72-5efe-493b-8087-59f5f3ec86a3", "cluster-id": "default" run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 [2024/05/04 11:15:21.926 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.93832ms"] [query="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/04 11:15:21.926 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq4] wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/batch_update_to_no_batch/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/04 11:15:22.440 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.366014ms"] [query="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/04 11:15:22.440 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts0] wait process cdc.test exit for 3-th time... table partition_table.finish_mark not exists for 3-th check, retry later [2024/05/04 11:15:22.955 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.799334ms"] [query="INSERT IGNORE INTO accounts0 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/04 11:15:22.955 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts1] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:15:23 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 11:15:23.469 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.394361ms"] [query="INSERT IGNORE INTO accounts1 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/04 11:15:23.469 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts2] capture_id: d39b3a72-5efe-493b-8087-59f5f3ec86a3 table ddl_manager.finish_mark not exists for 63-th check, retry later check diff failed 6-th time, retry later wait process 2689 exit for 1-th time... go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:23 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49546b50-babd-4fe0-bf70-3c3afa0574d8 {"id":"49546b50-babd-4fe0-bf70-3c3afa0574d8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41991d79cc 49546b50-babd-4fe0-bf70-3c3afa0574d8 /tidb/cdc/default/default/upstream/7364977736592798680 {"id":7364977736592798680,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49546b50-babd-4fe0-bf70-3c3afa0574d8 {"id":"49546b50-babd-4fe0-bf70-3c3afa0574d8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41991d79cc 49546b50-babd-4fe0-bf70-3c3afa0574d8 /tidb/cdc/default/default/upstream/7364977736592798680 {"id":7364977736592798680,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49546b50-babd-4fe0-bf70-3c3afa0574d8 {"id":"49546b50-babd-4fe0-bf70-3c3afa0574d8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41991d79cc 49546b50-babd-4fe0-bf70-3c3afa0574d8 /tidb/cdc/default/default/upstream/7364977736592798680 {"id":7364977736592798680,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/04 11:15:23.983 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.186838ms"] [query="INSERT IGNORE INTO accounts2 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/04 11:15:23.983 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts3] check diff failed 3-th time, retry later wait process 2689 exit for 2-th time... [2024/05/04 11:15:24.496 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.134397ms"] [query="INSERT IGNORE INTO accounts3 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/04 11:15:24.497 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts4] /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2689) - No such process wait process 2689 exit for 3-th time... process 2689 already exit cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep d39b3a72-5efe-493b-8087-59f5f3ec86a3 -A1 | grep '"is-owner": true' table partition_table.finish_mark not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/batch_update_to_no_batch Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0665b0840013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:7623, start at 2024-05-04 11:15:24.881128044 +0800 CST m=+5.931054901 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:24.891 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:24.883 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:24.883 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 11:15:25.010 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.379303ms"] [query="INSERT IGNORE INTO accounts4 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 table ddl_manager.finish_mark not exists for 64-th check, retry later go: downloading github.com/jmespath/go-jmespath v0.4.0 check diff successfully check diff failed 1-th time, retry later table consistent_replicate_gbk.GBKTABLE not exists for 1-th check, retry later check diff failed 4-th time, retry later go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd "is-owner": true, run task successfully test_kill_owner: pass wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0665b0840013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:7623, start at 2024-05-04 11:15:24.881128044 +0800 CST m=+5.931054901 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:24.891 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:24.883 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:24.883 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0665b19c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:7703, start at 2024-05-04 11:15:24.940284257 +0800 CST m=+5.914186054 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:24.950 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:24.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:24.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table partition_table.finish_mark not exists for 5-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 65-th check, retry later [2024/05/04 11:15:27.519 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table consistent_replicate_gbk.GBKTABLE exists table consistent_replicate_gbk.check1 not exists for 1-th check, retry later check diff failed 5-th time, retry later wait process cdc.test exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_hang_up_owner [Sat May 4 11:15:28 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.30613063.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table testSync.usertable exists table testSync.simple1 exists table testSync.simple2 exists wait process cdc.test exit for 2-th time... table partition_table.finish_mark not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 66-th check, retry later check diff successfully Starting Upstream TiDB... [2024/05/04 11:15:29.532 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] wait process cdc.test exit for 3-th time... table consistent_replicate_gbk.check1 not exists for 2-th check, retry later Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9078.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:15:29 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... check diff failed 6-th time, retry later wait process cdc.test exit for 3-th time... table partition_table.finish_mark not exists for 7-th check, retry later + set +x + tso='449522572901220354 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522572901220354 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:15:31 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.91179119.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 67-th check, retry later wait process cdc.test exit for 4-th time... table consistent_replicate_gbk.check1 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00256ce10 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522573227851782 MinTableBarrierTs:449522573241221121 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522573241221121, checkpointTs: 449522573241221121, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522573227851782, checkpointTs: 449522573227851782, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522573227851782, checkpointTs: 449522573227851782, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b248305-9787-4659-a95c-1098c733bb96 {"id":"9b248305-9787-4659-a95c-1098c733bb96","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792528} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32fea 9b248305-9787-4659-a95c-1098c733bb96 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522573227851782,"min-table-barrier-ts":449522573241221121,"admin-job-type":0} /tidb/cdc/default/default/task/position/9b248305-9787-4659-a95c-1098c733bb96/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00256ce10 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522573227851782 MinTableBarrierTs:449522573241221121 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522573241221121, checkpointTs: 449522573241221121, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522573227851782, checkpointTs: 449522573227851782, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522573227851782, checkpointTs: 449522573227851782, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b248305-9787-4659-a95c-1098c733bb96 {"id":"9b248305-9787-4659-a95c-1098c733bb96","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792528} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32fea 9b248305-9787-4659-a95c-1098c733bb96 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522573227851782,"min-table-barrier-ts":449522573241221121,"admin-job-type":0} /tidb/cdc/default/default/task/position/9b248305-9787-4659-a95c-1098c733bb96/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00256ce10 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522573227851782 MinTableBarrierTs:449522573241221121 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522573241221121, checkpointTs: 449522573241221121, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522573227851782, checkpointTs: 449522573227851782, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522573227851782, checkpointTs: 449522573227851782, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b248305-9787-4659-a95c-1098c733bb96 {"id":"9b248305-9787-4659-a95c-1098c733bb96","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792528} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32fea 9b248305-9787-4659-a95c-1098c733bb96 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522573227851782,"min-table-barrier-ts":449522573241221121,"admin-job-type":0} /tidb/cdc/default/default/task/position/9b248305-9787-4659-a95c-1098c733bb96/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 + grep -q 'etcd info' {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' check diff successfully cdc.test: no process found wait process cdc.test exit for 5-th time... process cdc.test already exit [Sat May 4 11:15:31 CST 2024] <<<<<< run test case owner_resign success! >>>>>> [2024/05/04 11:15:31.544 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] check diff failed 7-th time, retry later wait process cdc.test exit for 1-th time... table partition_table.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... "is-owner": true, run task successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:15:33 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.58925894.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 68-th check, retry later [2024/05/04 11:15:33.558 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:34 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b51e81b-a3d7-4970-ab98-30dd37f50502 {"id":"6b51e81b-a3d7-4970-ab98-30dd37f50502","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792531} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41993760da 6b51e81b-a3d7-4970-ab98-30dd37f50502 /tidb/cdc/default/default/upstream/7364977766136310256 {"id":7364977766136310256,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b51e81b-a3d7-4970-ab98-30dd37f50502 {"id":"6b51e81b-a3d7-4970-ab98-30dd37f50502","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792531} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41993760da 6b51e81b-a3d7-4970-ab98-30dd37f50502 /tidb/cdc/default/default/upstream/7364977766136310256 {"id":7364977766136310256,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b51e81b-a3d7-4970-ab98-30dd37f50502 {"id":"6b51e81b-a3d7-4970-ab98-30dd37f50502","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792531} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41993760da 6b51e81b-a3d7-4970-ab98-30dd37f50502 /tidb/cdc/default/default/upstream/7364977766136310256 {"id":7364977766136310256,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9177.out cli changefeed create --start-ts=449522572901220354 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table partition_table.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) owner pid: 3066 owner id 9b248305-9787-4659-a95c-1098c733bb96 [Sat May 4 11:15:34 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.31853187.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' table ddl_manager.finish_mark not exists for 69-th check, retry later check diff failed 8-th time, retry later + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:15:35.569 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] Create changefeed successfully! ID: 0290219c-af09-4cf2-b1c6-b16dc0a9c931 Info: {"upstream_id":7364977766136310256,"namespace":"default","id":"0290219c-af09-4cf2-b1c6-b16dc0a9c931","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:15:35.997534722+08:00","start_ts":449522572901220354,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522572901220354,"checkpoint_ts":449522572901220354,"checkpoint_time":"2024-05-04 11:15:29.683"} PASS start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/0e72c5d3-67a5-4603-bcea-181845433b9e {UpstreamID:7364977736592798680 Namespace:default ID:0e72c5d3-67a5-4603-bcea-181845433b9e SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:23.92371564 +0800 CST StartTs:449522571360075779 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004058ea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522571386290179} {CheckpointTs:449522574400946178 MinTableBarrierTs:449522574400946178 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522574400946178, checkpointTs: 449522574400946178, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522574400946178, checkpointTs: 449522574400946178, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a03c4ff-cb2b-458f-b673-a41bdc5faf7a {"id":"7a03c4ff-cb2b-458f-b673-a41bdc5faf7a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792533} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41991d7a8a 7a03c4ff-cb2b-458f-b673-a41bdc5faf7a /tidb/cdc/default/default/changefeed/info/0e72c5d3-67a5-4603-bcea-181845433b9e {"upstream-id":7364977736592798680,"namespace":"default","changefeed-id":"0e72c5d3-67a5-4603-bcea-181845433b9e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:23.92371564+08:00","start-ts":449522571360075779,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522571386290179} /tidb/cdc/default/default/changefeed/status/0e72c5d3-67a5-4603-bcea-181845433b9e {"checkpoint-ts":449522574400946178,"min-table-barrier-ts":449522574400946178,"admin-job-type":0} /tidb/cdc/default/default/task/position/7a03c4ff-cb2b-458f-b673-a41bdc5faf7a/0e72c5d3-67a5-4603-bcea-181845433b9e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977736592798680 {"id":7364977736592798680,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/0e72c5d3-67a5-4603-bcea-181845433b9e {UpstreamID:7364977736592798680 Namespace:default ID:0e72c5d3-67a5-4603-bcea-181845433b9e SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:23.92371564 +0800 CST StartTs:449522571360075779 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004058ea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522571386290179} {CheckpointTs:449522574400946178 MinTableBarrierTs:449522574400946178 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522574400946178, checkpointTs: 449522574400946178, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522574400946178, checkpointTs: 449522574400946178, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a03c4ff-cb2b-458f-b673-a41bdc5faf7a {"id":"7a03c4ff-cb2b-458f-b673-a41bdc5faf7a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792533} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41991d7a8a 7a03c4ff-cb2b-458f-b673-a41bdc5faf7a /tidb/cdc/default/default/changefeed/info/0e72c5d3-67a5-4603-bcea-181845433b9e {"upstream-id":7364977736592798680,"namespace":"default","changefeed-id":"0e72c5d3-67a5-4603-bcea-181845433b9e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:23.92371564+08:00","start-ts":449522571360075779,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522571386290179} /tidb/cdc/default/default/changefeed/status/0e72c5d3-67a5-4603-bcea-181845433b9e {"checkpoint-ts":449522574400946178,"min-table-barrier-ts":449522574400946178,"admin-job-type":0} /tidb/cdc/default/default/task/position/7a03c4ff-cb2b-458f-b673-a41bdc5faf7a/0e72c5d3-67a5-4603-bcea-181845433b9e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977736592798680 {"id":7364977736592798680,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/0e72c5d3-67a5-4603-bcea-181845433b9e {UpstreamID:7364977736592798680 Namespace:default ID:0e72c5d3-67a5-4603-bcea-181845433b9e SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:23.92371564 +0800 CST StartTs:449522571360075779 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004058ea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522571386290179} {CheckpointTs:449522574400946178 MinTableBarrierTs:449522574400946178 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522574400946178, checkpointTs: 449522574400946178, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522574400946178, checkpointTs: 449522574400946178, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a03c4ff-cb2b-458f-b673-a41bdc5faf7a {"id":"7a03c4ff-cb2b-458f-b673-a41bdc5faf7a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792533} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41991d7a8a 7a03c4ff-cb2b-458f-b673-a41bdc5faf7a /tidb/cdc/default/default/changefeed/info/0e72c5d3-67a5-4603-bcea-181845433b9e {"upstream-id":7364977736592798680,"namespace":"default","changefeed-id":"0e72c5d3-67a5-4603-bcea-181845433b9e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:23.92371564+08:00","start-ts":449522571360075779,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522571386290179} /tidb/cdc/default/default/changefeed/status/0e72c5d3-67a5-4603-bcea-181845433b9e {"checkpoint-ts":449522574400946178,"min-table-barrier-ts":449522574400946178,"admin-job-type":0} /tidb/cdc/default/default/task/position/7a03c4ff-cb2b-458f-b673-a41bdc5faf7a/0e72c5d3-67a5-4603-bcea-181845433b9e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"wa+ grep -q 'etcd info' rning":null} /tidb/cdc/default/default/upstream/7364977736592798680 {"id":7364977736592798680,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06664dd00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx, pid:3784, start at 2024-05-04 11:15:34.938957861 +0800 CST m=+5.399073107 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:34.945 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:34.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:34.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06664dd00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx, pid:3784, start at 2024-05-04 11:15:34.938957861 +0800 CST m=+5.399073107 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:34.945 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:34.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:34.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06664e68000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx, pid:3874, start at 2024-05-04 11:15:34.954455017 +0800 CST m=+5.332049173 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:34.963 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:34.938 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:34.938 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table partition_table.finish_mark not exists for 10-th check, retry later table ddl_manager.finish_mark not exists for 70-th check, retry later [2024/05/04 11:15:37.582 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 9-th time, retry later + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:38 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031681b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522575075966978 MinTableBarrierTs:449522575075966981 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b248305-9787-4659-a95c-1098c733bb96 {"id":"9b248305-9787-4659-a95c-1098c733bb96","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792528} /tidb/cdc/default/__cdc_meta__/capture/ff56266d-7adb-4660-b045-e0db8787b9ad {"id":"ff56266d-7adb-4660-b045-e0db8787b9ad","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792536} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32fea 9b248305-9787-4659-a95c-1098c733bb96 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33055 ff56266d-7adb-4660-b045-e0db8787b9ad /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522575075966978,"min-table-barrier-ts":449522575075966981,"admin-job-type":0} /tidb/cdc/default/default/task/position/9b248305-9787-4659-a95c-1098c733bb96/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ff56266d-7adb-4660-b045-e0db8787b9ad/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031681b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522575075966978 MinTableBarrierTs:449522575075966981 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b248305-9787-4659-a95c-1098c733bb96 {"id":"9b248305-9787-4659-a95c-1098c733bb96","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792528} /tidb/cdc/default/__cdc_meta__/capture/ff56266d-7adb-4660-b045-e0db8787b9ad {"id":"ff56266d-7adb-4660-b045-e0db8787b9ad","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792536} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32fea 9b248305-9787-4659-a95c-1098c733bb96 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33055 ff56266d-7adb-4660-b045-e0db8787b9ad /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522575075966978,"min-table-barrier-ts":449522575075966981,"admin-job-type":0} /tidb/cdc/default/default/task/position/9b248305-9787-4659-a95c-1098c733bb96/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ff56266d-7adb-4660-b045-e0db8787b9ad/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031681b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522575075966978 MinTableBarrierTs:449522575075966981 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b248305-9787-4659-a95c-1098c733bb96 {"id":"9b248305-9787-4659-a95c-1098c733bb96","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792528} /tidb/cdc/default/__cdc_meta__/capture/ff56266d-7adb-4660-b045-e0db8787b9ad {"id":"ff56266d-7adb-4660-b045-e0db8787b9ad","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792536} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e32fea 9b248305-9787-4659-a95c-1098c733bb96 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33055 ff56266d-7adb-4660-b045-e0db8787b9ad /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522575075966978,"min-table-barrier-ts":449522575075966981,"admin-job-type":0} /tidb/cdc/default/default/task/position/9b248305-9787-4659-a95c-1098c733bb96/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ff56266d-7adb-4660-b045-e0db8787b9ad/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "9b248305-9787-4659-a95c-1098c733bb96" | grep id table partition_table.finish_mark not exists for 11-th check, retry later table ddl_manager.finish_mark not exists for 71-th check, retry later [2024/05/04 11:15:39.594 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 10-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release "cluster-id": "default" "id": "ff56266d-7adb-4660-b045-e0db8787b9ad", "cluster-id": "default" run task successfully table ddl_sequence.finish_mark not exists for 1-th check, retry later table partition_table.finish_mark exists check diff successfully [Sat May 4 11:15:40 CST 2024] <<<<<< START cdc server in batch_update_to_no_batch case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.52625264.out server --log-file /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 72-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 11-th time, retry later [2024/05/04 11:15:41.612 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [2024/05/04 11:15:41.628 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark] [ts=449522575792668677] [2024/05/04 11:15:41.628 +08:00] [INFO] [case.go:498] ["all tables synced"] [endTs=449522575792668677] wait process cdc.test exit for 2-th time... capture_id: ff56266d-7adb-4660-b045-e0db8787b9ad ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep 'ff56266d-7adb-4660-b045-e0db8787b9ad' ff56266d-7adb-4660-b045-e0db8787b9ad run task successfully test_hang_up_owner: pass Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:15:42 CST 2024] <<<<<< run test case partition_table success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_sequence.finish_mark not exists for 2-th check, retry later check_redo_resolved_ts 0e72c5d3-67a5-4603-bcea-181845433b9e 449522576144203778 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/0e72c5d3-67a5-4603-bcea-181845433b9e/meta global resolved ts 449522574663090207 not forward to 449522576144203778 run task failed 1-th time, retry later wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 73-th check, retry later wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666d0b0001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2, pid:3844, start at 2024-05-04 11:15:43.296203962 +0800 CST m=+5.855604211 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:43.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:43.276 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:43.276 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_redo_resolved_ts 0e72c5d3-67a5-4603-bcea-181845433b9e 449522576144203778 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/0e72c5d3-67a5-4603-bcea-181845433b9e/meta cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_expire_owner [Sat May 4 11:15:44 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.33443346.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 12-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:44 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1e624dd-8d37-4f93-a4f9-2d9ad4a17fc2 {"id":"b1e624dd-8d37-4f93-a4f9-2d9ad4a17fc2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792541} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41996a98d2 b1e624dd-8d37-4f93-a4f9-2d9ad4a17fc2 /tidb/cdc/default/default/upstream/7364977820727552109 {"id":7364977820727552109,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1e624dd-8d37-4f93-a4f9-2d9ad4a17fc2 {"id":"b1e624dd-8d37-4f93-a4f9-2d9ad4a17fc2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792541} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41996a98d2 b1e624dd-8d37-4f93-a4f9-2d9ad4a17fc2 /tidb/cdc/default/default/upstream/7364977820727552109 {"id":7364977820727552109,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1e624dd-8d37-4f93-a4f9-2d9ad4a17fc2 {"id":"b1e624dd-8d37-4f93-a4f9-2d9ad4a17fc2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792541} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41996a98d2 b1e624dd-8d37-4f93-a4f9-2d9ad4a17fc2 /tidb/cdc/default/default/upstream/7364977820727552109 {"id":7364977820727552109,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5320.out cli changefeed create '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' -c test run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Create changefeed successfully! ID: test Info: {"upstream_id":7364977820727552109,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-04T11:15:44.511139423+08:00","start_ts":449522576758931461,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522576758931461,"checkpoint_ts":449522576758931461,"checkpoint_time":"2024-05-04 11:15:44.399"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... table ddl_sequence.finish_mark not exists for 3-th check, retry later build mysql_test successfully run all mysql test cases time="2024-05-04T11:15:41+08:00" level=info msg="running tests: [alter_table bigint composite_index concurrent_ddl date_formats drop mysql_replace]" time="2024-05-04T11:15:41+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000126000 {{} {} 0} {0 0} [0xc000314000] map[] 0 1 0xc00012a000 false map[0xc000314000:map[0xc000314000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [2024/05/04 11:15:45.101 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark1000] [2024/05/04 11:15:45.101 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark1000] wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 74-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666d0b0001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2, pid:3844, start at 2024-05-04 11:15:43.296203962 +0800 CST m=+5.855604211 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:43.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:43.276 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:43.276 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666d0980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2, pid:3928, start at 2024-05-04 11:15:43.301480007 +0800 CST m=+5.785161125 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:43.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:43.270 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:43.270 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process cdc.test exit for 3-th time... check diff failed 13-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 11:15:46.415 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/04 11:15:46.416 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/04 11:15:46.417 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/04 11:15:46.418 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/04 11:15:46.420 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/04 11:15:46.420 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=449522574663090207] [resolvedTs=449522576498098181] [2024/05/04 11:15:46.421 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/04 11:15:46.421 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/04 11:15:46.421 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 11:15:46.422 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 11:15:46.422 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 11:15:46.424 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=3.69742ms] [2024/05/04 11:15:46.424 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=3.788071ms] [2024/05/04 11:15:46.426 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 11:15:46.428 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/04 11:15:46.429 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/04 11:15:46.429 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 11:15:46.429 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 11:15:46.429 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 11:15:46.432 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 11:15:46.434 +08:00] [INFO] [mysql_ddl_sink.go:107] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/04 11:15:46.434 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449522574663090202,\"CommitTs\":449522574663090207,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_gbk\",\"Table\":\"GBKTABLE2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 11:15:46.435 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [commitTs=449522574663090207] [namespace=] [changefeed=] [2024/05/04 11:15:46.436 +08:00] [INFO] [mysql_ddl_sink.go:127] ["Execute DDL failed, but error can be ignored"] [startTs=449522574663090202] [ddl="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [namespace=] [changefeed=] [error="Error 1050 (42S01): Table 'consistent_replicate_gbk.GBKTABLE2' already exists"] ./t/alter_table.test: ok! 64 test cases passed, take time 5.159375866 s time="2024-05-04T11:15:46+08:00" level=info msg="run test [alter_table] ok" time="2024-05-04T11:15:46+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000308390 {{} {} 0} {0 0} [0xc0000fecf0] map[] 0 1 0xc000300de0 false map[0xc0000fecf0:map[0xc0000fecf0:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 11:15:46.637 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449522576498098181] [2024/05/04 11:15:46.637 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449522576498098181] [2024/05/04 11:15:46.737 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=1] [appliedDDLCount=1] [currentCheckpoint=449522576498098181] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/04 11:15:46.738 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/04 11:15:46.738 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/04 11:15:46.737 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/04 11:15:46.738 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] Apply redo log successfully check diff successfully [Sat May 4 11:15:46 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT table ddl_sequence.finish_mark not exists for 4-th check, retry later [2024/05/04 11:15:47.101 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] [2024/05/04 11:15:47.204 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark1000] [ts=449522577378639877] [2024/05/04 11:15:47.204 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark1000] [2024/05/04 11:15:47.204 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522577378639877] [2024/05/04 11:15:47.207 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:15:47.211 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:15:47.301 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:15:47.310 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5361.out cli changefeed pause -c test table ddl_manager.finish_mark not exists for 75-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001797950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522577173381125 MinTableBarrierTs:449522577435262981 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522577435262981, checkpointTs: 449522577435262981, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522577435262978, checkpointTs: 449522577435262978, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522577435262978, checkpointTs: 449522577435262978, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e84cf82-c38a-43e9-b82e-dac3906442d4 {"id":"0e84cf82-c38a-43e9-b82e-dac3906442d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792544} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e330bd 0e84cf82-c38a-43e9-b82e-dac3906442d4 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522577435262978,"min-table-barrier-ts":449522577435262981,"admin-job-type":0} /tidb/cdc/default/default/task/position/0e84cf82-c38a-43e9-b82e-dac3906442d4/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001797950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522577173381125 MinTableBarrierTs:449522577435262981 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522577435262981, checkpointTs: 449522577435262981, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522577435262978, checkpointTs: 449522577435262978, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522577435262978, checkpointTs: 449522577435262978, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e84cf82-c38a-43e9-b82e-dac3906442d4 {"id":"0e84cf82-c38a-43e9-b82e-dac3906442d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792544} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e330bd 0e84cf82-c38a-43e9-b82e-dac3906442d4 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522577435262978,"min-table-barrier-ts":449522577435262981,"admin-job-type":0} /tidb/cdc/default/default/task/position/0e84cf82-c38a-43e9-b82e-dac3906442d4/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001797950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522577173381125 MinTableBarrierTs:449522577435262981 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522577435262981, checkpointTs: 449522577435262981, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522577435262978, checkpointTs: 449522577435262978, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522577435262978, checkpointTs: 449522577435262978, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e84cf82-c38a-43e9-b82e-dac3906442d4 {"id":"0e84cf82-c38a-43e9-b82e-dac3906442d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792544} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e330bd 0e84cf82-c38a-43e9-b82e-dac3906442d4 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522577435262978,"min-table-barrier-ts":449522577435262981,"admin-job-type":0} /tidb/cdc/default/default/task/position/0e84cf82-c38a-43e9-b82e-dac3906442d4/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [Sat May 4 11:15:47 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.52505252.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:15:47.707 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark2000] [2024/05/04 11:15:47.707 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark2000] PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 14-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066714400004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:10803, start at 2024-05-04 11:15:47.601121613 +0800 CST m=+5.341653865 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:47.608 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:47.600 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:47.600 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 5-th check, retry later "is-owner": true, run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5397.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=false' --no-confirm table ddl_manager.finish_mark not exists for 76-th check, retry later [2024/05/04 11:15:49.708 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=true To:mysql://root@127.0.0.1:3306/?batch-dml-enable=false} Update changefeed config successfully! ID: test Info: {"upstream_id":7364977820727552109,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=false","create_time":"2024-05-04T11:15:44.511139423+08:00","start_ts":449522576758931461,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":0,"checkpoint_ts":449522577243897870,"checkpoint_time":"2024-05-04 11:15:46.249"} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066714400004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:10803, start at 2024-05-04 11:15:47.601121613 +0800 CST m=+5.341653865 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:47.608 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:47.600 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:47.600 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066714040013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:10889, start at 2024-05-04 11:15:47.611773376 +0800 CST m=+5.283352215 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:47.618 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:47.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:47.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 15-th time, retry later table ddl_sequence.finish_mark not exists for 6-th check, retry later owner pid: 3349 owner id 0e84cf82-c38a-43e9-b82e-dac3906442d4 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '0e84cf82-c38a-43e9-b82e-dac3906442d4' /tidb/cdc/default/__cdc_meta__/owner/22318f4198e330bd run task successfully process status: S cdc.test cli capture list 2>&1 | grep '"is-owner": true' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 11:15:45.705 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 20"] [2024/05/04 11:15:50.701 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 28"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:50 GMT < Content-Length: 853 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a79b520-44b0-4170-a747-828b2763d739 {"id":"2a79b520-44b0-4170-a747-828b2763d739","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792548} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199837e8d 2a79b520-44b0-4170-a747-828b2763d739 /tidb/cdc/default/default/upstream/7364977855191071452 {"id":7364977855191071452,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a79b520-44b0-4170-a747-828b2763d739 {"id":"2a79b520-44b0-4170-a747-828b2763d739","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792548} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199837e8d 2a79b520-44b0-4170-a747-828b2763d739 /tidb/cdc/default/default/upstream/7364977855191071452 {"id":7364977855191071452,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a79b520-44b0-4170-a747-828b2763d739 {"id":"2a79b520-44b0-4170-a747-828b2763d739","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792548} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199837e8d 2a79b520-44b0-4170-a747-828b2763d739 /tidb/cdc/default/default/upstream/7364977855191071452 {"id":7364977855191071452,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 77-th check, retry later Create changefeed successfully! ID: 22f3030f-4298-452a-b703-563096547e3a Info: {"upstream_id":7364977855191071452,"namespace":"default","id":"22f3030f-4298-452a-b703-563096547e3a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:15:51.120312344+08:00","start_ts":449522578482790404,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522578482790404,"checkpoint_ts":449522578482790404,"checkpoint_time":"2024-05-04 11:15:50.975"} split_and_random_merge scale: 20 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5433.out cli changefeed resume -c test PASS "is-owner": true, run task successfully test_expire_owner pass [2024/05/04 11:15:51.721 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] [2024/05/04 11:15:51.808 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark2000] [ts=449522578253152261] [2024/05/04 11:15:51.808 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark2000] [2024/05/04 11:15:51.808 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522578253152261] [2024/05/04 11:15:51.822 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:15:51.825 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:15:51.830 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:15:51.838 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:15:52.020 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark3000] [2024/05/04 11:15:52.020 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark3000] coverage: 2.1% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... check diff failed 16-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066752c40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6, pid:4207, start at 2024-05-04 11:15:51.601734495 +0800 CST m=+5.260019249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:51.608 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:51.601 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:51.601 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066752c40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6, pid:4207, start at 2024-05-04 11:15:51.601734495 +0800 CST m=+5.260019249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:51.608 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:51.601 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:51.601 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066752b00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6, pid:4294, start at 2024-05-04 11:15:51.613169577 +0800 CST m=+5.225050770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:51.620 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:51.596 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:51.596 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... \033[0;36m<<< Run all test success >>>\033[0m ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc already exists) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12138.out cli tso query --pd=http://127.0.0.1:2379 ./t/bigint.test: ok! 138 test cases passed, take time 5.465095125 s time="2024-05-04T11:15:51+08:00" level=info msg="run test [bigint] ok" time="2024-05-04T11:15:51+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0000ca7d0 {{} {} 0} {0 0} [0xc0003a4a20] map[] 0 1 0xc0003ae180 false map[0xc0003a4a20:map[0xc0003a4a20:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" ./t/composite_index.test: ok! 12 test cases passed, take time 0.26763566 s time="2024-05-04T11:15:52+08:00" level=info msg="run test [composite_index] ok" time="2024-05-04T11:15:52+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000126028 {{} {} 0} {0 0} [0xc0000fe090] map[] 0 1 0xc0004000c0 false map[0xc0000fe090:map[0xc0000fe090:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" ./t/concurrent_ddl.test: ok! 8 test cases passed, take time 0.426397571 s time="2024-05-04T11:15:52+08:00" level=info msg="run test [concurrent_ddl] ok" time="2024-05-04T11:15:52+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc00038c040 {{} {} 0} {0 0} [0xc0000fe2d0] map[] 0 1 0xc0003802a0 false map[0xc0000fe2d0:map[0xc0000fe2d0:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [Pipeline] // cache table ddl_sequence.finish_mark not exists for 7-th check, retry later [Pipeline] } [Pipeline] // dir [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // withCredentials [Pipeline] } + set +x [Pipeline] // timeout [Pipeline] } table ddl_manager.finish_mark not exists for 78-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_owner_cleanup_stale_tasks [Sat May 4 11:15:53 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.35463548.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // withEnv ./t/date_formats.test: ok! 99 test cases passed, take time 1.042775533 s time="2024-05-04T11:15:53+08:00" level=info msg="run test [date_formats] ok" time="2024-05-04T11:15:53+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000126368 {{} {} 0} {0 0} [0xc000385c20] map[] 0 1 0xc000401260 false map[0xc000385c20:map[0xc000385c20:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [Pipeline] } [2024/05/04 11:15:54.020 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] [2024/05/04 11:15:54.037 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark3000] [ts=449522579223085062] [2024/05/04 11:15:54.037 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark3000] [2024/05/04 11:15:54.037 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522579223085062] [2024/05/04 11:15:54.039 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:15:54.042 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:15:54.103 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:15:54.110 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [Pipeline] // node [Pipeline] } + set +x + tso='449522579000262657 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522579000262657 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Pipeline] // podTemplate check diff failed 17-th time, retry later [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Sat May 4 11:15:54 CST 2024] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.1218612188.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5490.out cli changefeed pause -c test [2024/05/04 11:15:55.009 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark4000] [2024/05/04 11:15:55.009 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark4000] table ddl_sequence.finish_mark exists check diff successfully PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 79-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... ./t/drop.test: ok! 48 test cases passed, take time 1.999919198 s time="2024-05-04T11:15:56+08:00" level=info msg="run test [drop] ok" time="2024-05-04T11:15:56+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0002ac288 {{} {} 0} {0 0} [0xc000385e60] map[] 0 1 0xc0002b0ba0 false map[0xc000385e60:map[0xc000385e60:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" ./t/mysql_replace.test: ok! 16 test cases passed, take time 0.185141045 s time="2024-05-04T11:15:56+08:00" level=info msg="run test [mysql_replace] ok" Great, All tests passed mysqltest end table test.finish_mark not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5526.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' --no-confirm ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) *************************** 1. row *************************** count(distinct region_id): 2 check diff failed 18-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:15:56 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=false To:mysql://root@127.0.0.1:3306/?batch-dml-enable=true} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00247f4d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522579794558979 MinTableBarrierTs:449522579794558982 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522579794558982, checkpointTs: 449522579794558982, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522579794558979, checkpointTs: 449522579794558979, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522579794558979, checkpointTs: 449522579794558979, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79eeef74-1493-4976-8d8d-e8d4fc866d06 {"id":"79eeef74-1493-4976-8d8d-e8d4fc866d06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e3312a 79eeef74-1493-4976-8d8d-e8d4fc866d06 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522579794558979,"min-table-barrier-ts":449522579794558982,"admin-job-type":0} /tidb/cdc/default/default/task/position/79eeef74-1493-4976-8d8d-e8d4fc866d06/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00247f4d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522579794558979 MinTableBarrierTs:449522579794558982 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522579794558982, checkpointTs: 449522579794558982, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522579794558979, checkpointTs: 449522579794558979, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522579794558979, checkpointTs: 449522579794558979, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79eeef74-1493-4976-8d8d-e8d4fc866d06 {"id":"79eeef74-1493-4976-8d8d-e8d4fc866d06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e3312a 79eeef74-1493-4976-8d8d-e8d4fc866d06 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522579794558979,"min-table-barrier-ts":449522579794558982,"admin-job-type":0} /tidb/cdc/default/default/task/position/79eeef74-1493-4976-8d8d-e8d4fc866d06/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00247f4d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522579794558979 MinTableBarrierTs:449522579794558982 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522579794558982, checkpointTs: 449522579794558982, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522579794558979, checkpointTs: 449522579794558979, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522579794558979, checkpointTs: 449522579794558979, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/79eeef74-1493-4976-8d8d-e8d4fc866d06 {"id":"79eeef74-1493-4976-8d8d-e8d4fc866d06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e3312a 79eeef74-1493-4976-8d8d-e8d4fc866d06 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522579794558979,"min-table-barrier-ts":449522579794558982,"admin-job-type":0} /tidb/cdc/default/default/task/position/79eeef74-1493-4976-8d8d-e8d4fc866d06/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' Update changefeed config successfully! ID: test Info: {"upstream_id":7364977820727552109,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-04T11:15:44.511139423+08:00","start_ts":449522576758931461,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":0,"checkpoint_ts":449522577270112261,"checkpoint_time":"2024-05-04 11:15:46.349"} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 80-th check, retry later [2024/05/04 11:15:57.009 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:57 GMT < Content-Length: 853 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2118a62c-d254-4f8c-b373-dc7f30986bb0 {"id":"2118a62c-d254-4f8c-b373-dc7f30986bb0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41999e28aa 2118a62c-d254-4f8c-b373-dc7f30986bb0 /tidb/cdc/default/default/upstream/7364977875656973656 {"id":7364977875656973656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2118a62c-d254-4f8c-b373-dc7f30986bb0 {"id":"2118a62c-d254-4f8c-b373-dc7f30986bb0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41999e28aa 2118a62c-d254-4f8c-b373-dc7f30986bb0 /tidb/cdc/default/default/upstream/7364977875656973656 {"id":7364977875656973656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2118a62c-d254-4f8c-b373-dc7f30986bb0 {"id":"2118a62c-d254-4f8c-b373-dc7f30986bb0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41999e28aa 2118a62c-d254-4f8c-b373-dc7f30986bb0 /tidb/cdc/default/default/upstream/7364977875656973656 {"id":7364977875656973656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12251.out cli changefeed create --start-ts=449522579000262657 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7364977875656973656,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:15:58.123797845+08:00","start_ts":449522579000262657,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522579000262657,"checkpoint_ts":449522579000262657,"checkpoint_time":"2024-05-04 11:15:52.949"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 2-th check, retry later [2024/05/04 11:15:58.210 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark5000] [2024/05/04 11:15:58.210 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark5000] "is-owner": true, run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5559.out cli changefeed resume -c test check diff failed 19-th time, retry later [2024/05/04 11:15:59.022 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] [Sat May 4 11:15:58 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.55515553.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:15:59.104 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark4000] [ts=449522580166803461] [2024/05/04 11:15:59.104 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark4000] [2024/05/04 11:15:59.104 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522580166803461] [2024/05/04 11:15:59.107 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:15:59.110 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:15:59.115 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:15:59.120 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] PASS table ddl_manager.finish_mark not exists for 81-th check, retry later + set +x coverage: 2.1% of statements in github.com/pingcap/tiflow/... table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later owner pid: 3551 owner id 79eeef74-1493-4976-8d8d-e8d4fc866d06 [Sat May 4 11:15:59 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.36613663.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:16:00.210 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] check diff successfully table test.finish_mark not exists for 3-th check, retry later + set +x table batch_update_to_no_batch.v1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0667c388000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff, pid:4068, start at 2024-05-04 11:15:58.82598222 +0800 CST m=+8.270611506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:58.835 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:58.818 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:58.818 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0667c388000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff, pid:4068, start at 2024-05-04 11:15:58.82598222 +0800 CST m=+8.270611506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:58.835 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:58.818 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:58.818 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0667c36c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-w435w-vg0r5-mj8ff, pid:4153, start at 2024-05-04 11:15:58.825344494 +0800 CST m=+8.210635957 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:58.836 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:58.811 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:58.811 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table test.`simple-dash` not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:01 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52c215b4-2762-4872-a528-e4631ec8a485 {"id":"52c215b4-2762-4872-a528-e4631ec8a485","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792558} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199b21c24 52c215b4-2762-4872-a528-e4631ec8a485 /tidb/cdc/default/default/upstream/7364977894412930621 {"id":7364977894412930621,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52c215b4-2762-4872-a528-e4631ec8a485 {"id":"52c215b4-2762-4872-a528-e4631ec8a485","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792558} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199b21c24 52c215b4-2762-4872-a528-e4631ec8a485 /tidb/cdc/default/default/upstream/7364977894412930621 {"id":7364977894412930621,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52c215b4-2762-4872-a528-e4631ec8a485 {"id":"52c215b4-2762-4872-a528-e4631ec8a485","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792558} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199b21c24 52c215b4-2762-4872-a528-e4631ec8a485 /tidb/cdc/default/default/upstream/7364977894412930621 {"id":7364977894412930621,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5619.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: af0c2b48-4d9f-4aa8-a14b-89c1ed970294 Info: {"upstream_id":7364977894412930621,"namespace":"default","id":"af0c2b48-4d9f-4aa8-a14b-89c1ed970294","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:16:01.721616921+08:00","start_ts":449522581268332549,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522581268332549,"checkpoint_ts":449522581268332549,"checkpoint_time":"2024-05-04 11:16:01.601"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark exists check diff successfully wait process cdc.test exit for 2-th time... split_and_random_merge scale: 40 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... wait process cdc.test exit for 2-th time... [2024/05/04 11:16:02.224 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] [2024/05/04 11:16:02.227 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark6000] [2024/05/04 11:16:02.227 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark6000] [2024/05/04 11:16:02.308 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark5000] [ts=449522580979449861] [2024/05/04 11:16:02.308 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark5000] [2024/05/04 11:16:02.308 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522580979449861] [2024/05/04 11:16:02.323 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:02.326 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:02.332 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:02.403 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table test.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:02 CST 2024] <<<<<< run test case clustered_index success! >>>>>> [Sat May 4 11:16:02 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.54485450.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003117560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522581367685127 MinTableBarrierTs:449522581367685131 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/778a73b8-e467-41a2-a292-6d2704ec49c8 {"id":"778a73b8-e467-41a2-a292-6d2704ec49c8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792560} /tidb/cdc/default/__cdc_meta__/capture/79eeef74-1493-4976-8d8d-e8d4fc866d06 {"id":"79eeef74-1493-4976-8d8d-e8d4fc866d06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e3312a 79eeef74-1493-4976-8d8d-e8d4fc866d06 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33179 778a73b8-e467-41a2-a292-6d2704ec49c8 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522581367685127,"min-table-barrier-ts":449522581367685131,"admin-job-type":0} /tidb/cdc/default/default/task/position/778a73b8-e467-41a2-a292-6d2704ec49c8/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/79eeef74-1493-4976-8d8d-e8d4fc866d06/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003117560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522581367685127 MinTableBarrierTs:449522581367685131 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/778a73b8-e467-41a2-a292-6d2704ec49c8 {"id":"778a73b8-e467-41a2-a292-6d2704ec49c8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792560} /tidb/cdc/default/__cdc_meta__/capture/79eeef74-1493-4976-8d8d-e8d4fc866d06 {"id":"79eeef74-1493-4976-8d8d-e8d4fc866d06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e3312a 79eeef74-1493-4976-8d8d-e8d4fc866d06 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33179 778a73b8-e467-41a2-a292-6d2704ec49c8 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522581367685127,"min-table-barrier-ts":449522581367685131,"admin-job-type":0} /tidb/cdc/default/default/task/position/778a73b8-e467-41a2-a292-6d2704ec49c8/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/79eeef74-1493-4976-8d8d-e8d4fc866d06/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003117560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522581367685127 MinTableBarrierTs:449522581367685131 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/778a73b8-e467-41a2-a292-6d2704ec49c8 {"id":"778a73b8-e467-41a2-a292-6d2704ec49c8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792560} /tidb/cdc/default/__cdc_meta__/capture/79eeef74-1493-4976-8d8d-e8d4fc866d06 {"id":"79eeef74-1493-4976-8d8d-e8d4fc866d06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e3312a 79eeef74-1493-4976-8d8d-e8d4fc866d06 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33179 778a73b8-e467-41a2-a292-6d2704ec49c8 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522581367685127,"min-table-barrier-ts":449522581367685131,"admin-job-type":0} /tidb/cdc/default/default/task/position/778a73b8-e467-41a2-a292-6d2704ec49c8/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/79eeef74-1493-4976-8d8d-e8d4fc866d06/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "79eeef74-1493-4976-8d8d-e8d4fc866d06" | grep id table batch_update_to_no_batch.v1 exists table batch_update_to_no_batch.recover_and_insert not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:03 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + set +x + info='{ "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522581726560260, "checkpoint_time": "2024-05-04 11:16:03.349", "error": null }' + echo '{ "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522581726560260, "checkpoint_time": "2024-05-04 11:16:03.349", "error": null }' { "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522581726560260, "checkpoint_time": "2024-05-04 11:16:03.349", "error": null } ++ echo '{' '"upstream_id":' 7364977875656973656, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522581726560260, '"checkpoint_time":' '"2024-05-04' '11:16:03.349",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364977875656973656, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522581726560260, '"checkpoint_time":' '"2024-05-04' '11:16:03.349",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] table batch_add_table.finish_mark not exists for 1-th check, retry later changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 [2024/05/04 11:16:04.299 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] [2024/05/04 11:16:04.400 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark6000] [ts=449522581870739460] [2024/05/04 11:16:04.400 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark6000] [2024/05/04 11:16:04.400 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522581870739460] [2024/05/04 11:16:04.403 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:04.410 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:04.414 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:04.505 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 "id": "778a73b8-e467-41a2-a292-6d2704ec49c8", "cluster-id": "default" "cluster-id": "default" run task successfully start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table test.finish_mark not exists for 5-th check, retry later table batch_update_to_no_batch.recover_and_insert not exists for 2-th check, retry later Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12660.out cli changefeed --changefeed-id custom-changefeed-name pause table batch_add_table.finish_mark exists capture_id: 778a73b8-e467-41a2-a292-6d2704ec49c8 check diff successfully PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:06 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1620bd04-5b12-4690-af80-30c9d886bea5 {"id":"1620bd04-5b12-4690-af80-30c9d886bea5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792563} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199bc8dd2 1620bd04-5b12-4690-af80-30c9d886bea5 /tidb/cdc/default/default/upstream/7364977910984283274 {"id":7364977910984283274,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1620bd04-5b12-4690-af80-30c9d886bea5 {"id":"1620bd04-5b12-4690-af80-30c9d886bea5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792563} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199bc8dd2 1620bd04-5b12-4690-af80-30c9d886bea5 /tidb/cdc/default/default/upstream/7364977910984283274 {"id":7364977910984283274,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1620bd04-5b12-4690-af80-30c9d886bea5 {"id":"1620bd04-5b12-4690-af80-30c9d886bea5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792563} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199bc8dd2 1620bd04-5b12-4690-af80-30c9d886bea5 /tidb/cdc/default/default/upstream/7364977910984283274 {"id":7364977910984283274,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/04 11:16:06.010 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark7000] [2024/05/04 11:16:06.010 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark7000] table test.finish_mark not exists for 6-th check, retry later wait process cdc.test exit for 1-th time... Verifying downstream PD is started... wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table batch_update_to_no_batch.recover_and_insert not exists for 3-th check, retry later wait process cdc.test exit for 3-th time... + set +x cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:07 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> [2024/05/04 11:16:08.010 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] [2024/05/04 11:16:08.025 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark7000] [ts=449522582919315466] [2024/05/04 11:16:08.025 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark7000] [2024/05/04 11:16:08.025 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522582919315466] [2024/05/04 11:16:08.034 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:08.037 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:08.052 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:08.103 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:08.500 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark8000] [2024/05/04 11:16:08.500 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark8000] *************************** 1. row *************************** count(distinct region_id): 1 table test.finish_mark not exists for 7-th check, retry later table batch_update_to_no_batch.recover_and_insert exists table batch_update_to_no_batch.finish_mark not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 0 [Sat May 4 11:16:08 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.38043806.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:16:10.501 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449522582250848260, "checkpoint_time": "2024-05-04 11:16:05.349", "error": null }' + echo '{ "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449522582250848260, "checkpoint_time": "2024-05-04 11:16:05.349", "error": null }' { "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449522582250848260, "checkpoint_time": "2024-05-04 11:16:05.349", "error": null } ++ echo '{' '"upstream_id":' 7364977875656973656, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449522582250848260, '"checkpoint_time":' '"2024-05-04' '11:16:05.349",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7364977875656973656, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449522582250848260, '"checkpoint_time":' '"2024-05-04' '11:16:05.349",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12755.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name table test.finish_mark not exists for 8-th check, retry later table batch_update_to_no_batch.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7364977875656973656,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:15:58.123797845+08:00","start_ts":449522579000262657,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":0,"checkpoint_ts":449522582250848260,"checkpoint_time":"2024-05-04 11:16:05.349"} PASS coverage: 2.8% of statements in github.com/pingcap/tiflow/... [2024/05/04 11:16:12.202 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark9000] [2024/05/04 11:16:12.202 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark9000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a259e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522582153854983 MinTableBarrierTs:449522582153854987 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/778a73b8-e467-41a2-a292-6d2704ec49c8 {"id":"778a73b8-e467-41a2-a292-6d2704ec49c8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792560} /tidb/cdc/default/__cdc_meta__/capture/79eeef74-1493-4976-8d8d-e8d4fc866d06 {"id":"79eeef74-1493-4976-8d8d-e8d4fc866d06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/capture/a7b0f612-9563-4f7f-bdf3-dab0283f9178 {"id":"a7b0f612-9563-4f7f-bdf3-dab0283f9178","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792569} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e3312a 79eeef74-1493-4976-8d8d-e8d4fc866d06 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33179 778a73b8-e467-41a2-a292-6d2704ec49c8 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e331ea a7b0f612-9563-4f7f-bdf3-dab0283f9178 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522582153854983,"min-table-barrier-ts":449522582153854987,"admin-job-type":0} /tidb/cdc/default/default/task/position/778a73b8-e467-41a2-a292-6d2704ec49c8/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/79eeef74-1493-4976-8d8d-e8d4fc866d06/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a7b0f612-9563-4f7f-bdf3-dab0283f9178/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a259e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522582153854983 MinTableBarrierTs:449522582153854987 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/778a73b8-e467-41a2-a292-6d2704ec49c8 {"id":"778a73b8-e467-41a2-a292-6d2704ec49c8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792560} /tidb/cdc/default/__cdc_meta__/capture/79eeef74-1493-4976-8d8d-e8d4fc866d06 {"id":"79eeef74-1493-4976-8d8d-e8d4fc866d06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/capture/a7b0f612-9563-4f7f-bdf3-dab0283f9178 {"id":"a7b0f612-9563-4f7f-bdf3-dab0283f9178","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792569} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e3312a 79eeef74-1493-4976-8d8d-e8d4fc866d06 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33179 778a73b8-e467-41a2-a292-6d2704ec49c8 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e331ea a7b0f612-9563-4f7f-bdf3-dab0283f9178 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522582153854983,"min-table-barrier-ts":449522582153854987,"admin-job-type":0} /tidb/cdc/default/default/task/position/778a73b8-e467-41a2-a292-6d2704ec49c8/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/79eeef74-1493-4976-8d8d-e8d4fc866d06/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a7b0f612-9563-4f7f-bdf3-dab0283f9178/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a259e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522582153854983 MinTableBarrierTs:449522582153854987 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/778a73b8-e467-41a2-a292-6d2704ec49c8 {"id":"778a73b8-e467-41a2-a292-6d2704ec49c8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792560} /tidb/cdc/default/__cdc_meta__/capture/79eeef74-1493-4976-8d8d-e8d4fc866d06 {"id":"79eeef74-1493-4976-8d8d-e8d4fc866d06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792554} /tidb/cdc/default/__cdc_meta__/capture/a7b0f612-9563-4f7f-bdf3-dab0283f9178 {"id":"a7b0f612-9563-4f7f-bdf3-dab0283f9178","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792569} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e3312a 79eeef74-1493-4976-8d8d-e8d4fc866d06 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33179 778a73b8-e467-41a2-a292-6d2704ec49c8 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e331ea a7b0f612-9563-4f7f-bdf3-dab0283f9178 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522582153854983,"min-table-barrier-ts":449522582153854987,"admin-job-type":0} /tidb/cdc/default/default/task/position/778a73b8-e467-41a2-a292-6d2704ec49c8/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/79eeef74-1493-4976-8d8d-e8d4fc866d06/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a7b0f612-9563-4f7f-bdf3-dab0283f9178/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' run task failed 1-th time, retry later [2024/05/04 11:16:12.516 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2024/05/04 11:16:12.602 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark8000] [ts=449522583705747466] [2024/05/04 11:16:12.602 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark8000] [2024/05/04 11:16:12.602 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522583705747466] [2024/05/04 11:16:12.604 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:12.608 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:12.612 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:12.618 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table test.finish_mark not exists for 9-th check, retry later split_and_random_merge scale: 80 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table processor_delay.t1 exists table processor_delay.t2 not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12792.out cli changefeed --changefeed-id custom-changefeed-name resume ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_update_to_no_batch.finish_mark not exists for 3-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' [2024/05/04 11:16:14.203 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] [2024/05/04 11:16:14.220 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark9000] [ts=449522584570822665] [2024/05/04 11:16:14.220 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark9000] [2024/05/04 11:16:14.220 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522584570822665] [2024/05/04 11:16:14.223 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:14.228 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:14.232 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:14.238 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table test.finish_mark not exists for 10-th check, retry later run task failed 2-th time, retry later + set +x table batch_update_to_no_batch.finish_mark not exists for 4-th check, retry later table processor_delay.t2 exists table processor_delay.t3 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: mysql... <<================= table test.finish_mark not exists for 11-th check, retry later [2024/05/04 11:16:16.427 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark10000] [2024/05/04 11:16:16.427 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark10000] table processor_delay.t3 exists table processor_delay.t4 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:15 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> table batch_update_to_no_batch.finish_mark exists check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522585396576261, "checkpoint_time": "2024-05-04 11:16:17.349", "error": null }' + echo '{ "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522585396576261, "checkpoint_time": "2024-05-04 11:16:17.349", "error": null }' { "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522585396576261, "checkpoint_time": "2024-05-04 11:16:17.349", "error": null } ++ echo '{' '"upstream_id":' 7364977875656973656, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522585396576261, '"checkpoint_time":' '"2024-05-04' '11:16:17.349",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364977875656973656, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522585396576261, '"checkpoint_time":' '"2024-05-04' '11:16:17.349",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12884.out cli changefeed --changefeed-id custom-changefeed-name remove wait process cdc.test exit for 2-th time... [2024/05/04 11:16:18.499 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/04 11:16:18.602 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark10000] [ts=449522585505103878] [2024/05/04 11:16:18.602 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark10000] [2024/05/04 11:16:18.602 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522585505103878] [2024/05/04 11:16:18.606 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:18.610 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:18.701 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:18.708 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449522585658720261 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' wait process cdc.test exit for 3-th time... table processor_delay.t4 exists table processor_delay.t5 not exists for 1-th check, retry later *************************** 1. row *************************** count(distinct region_id): 1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0668d3980002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q, pid:7127, start at 2024-05-04 11:16:16.231491213 +0800 CST m=+5.268752259 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:16.238 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:16.230 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:16.230 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0668d3980002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q, pid:7127, start at 2024-05-04 11:16:16.231491213 +0800 CST m=+5.268752259 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:16.238 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:16.230 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:16.230 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0668d5e40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9s9lq-mw7bz-l4s9q, pid:7200, start at 2024-05-04 11:16:16.386527119 +0800 CST m=+5.347041626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:16.393 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:16.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:16.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:19 CST 2024] <<<<<< run test case batch_update_to_no_batch success! >>>>>> [2024/05/04 11:16:19.520 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark11000] [2024/05/04 11:16:19.520 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark11000] + set +x "is-owner": true, run task successfully nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.finish_mark not exists for 12-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t5 exists table processor_delay.t6 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 11:16:21.520 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/04 11:16:21.614 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark11000] [ts=449522586379616261] [2024/05/04 11:16:21.614 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark11000] [2024/05/04 11:16:21.614 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522586379616261] [2024/05/04 11:16:21.699 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:21.710 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [Sat May 4 11:16:21 CST 2024] <<<<<< START cdc server in consistent_replicate_nfs case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_nfs.85218523.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:16:21.801 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:21.807 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table test.finish_mark not exists for 13-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t6 exists table processor_delay.t7 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 11:16:23.310 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark12000] [2024/05/04 11:16:23.310 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark12000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12965.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7364977875656973656,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:16:24.123563088+08:00","start_ts":449522587139833858,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522587139833858,"checkpoint_ts":449522587139833858,"checkpoint_time":"2024-05-04 11:16:23.999"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table test.finish_mark not exists for 14-th check, retry later table region_merge.t1 exists check diff failed 1-th time, retry later < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:24 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17156b5f-9223-4512-99a8-f7dfef470ce1 {"id":"17156b5f-9223-4512-99a8-f7dfef470ce1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792581} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a0858d1 17156b5f-9223-4512-99a8-f7dfef470ce1 /tidb/cdc/default/default/upstream/7364977988761995666 {"id":7364977988761995666,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17156b5f-9223-4512-99a8-f7dfef470ce1 {"id":"17156b5f-9223-4512-99a8-f7dfef470ce1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792581} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a0858d1 17156b5f-9223-4512-99a8-f7dfef470ce1 /tidb/cdc/default/default/upstream/7364977988761995666 {"id":7364977988761995666,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17156b5f-9223-4512-99a8-f7dfef470ce1 {"id":"17156b5f-9223-4512-99a8-f7dfef470ce1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792581} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a0858d1 17156b5f-9223-4512-99a8-f7dfef470ce1 /tidb/cdc/default/default/upstream/7364977988761995666 {"id":7364977988761995666,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ____________________________________ *************************** 1. row *************************** primary_ts: 449522523480522753 secondary_ts: 449522526164877327 *************************** 2. row *************************** primary_ts: 449522531344842752 secondary_ts: 449522531748282370 *************************** 3. row *************************** primary_ts: 449522539209162752 secondary_ts: 449522539350720513 *************************** 4. row *************************** primary_ts: 449522547073482752 secondary_ts: 449522547215040513 *************************** 5. row *************************** primary_ts: 449522554937802752 secondary_ts: 449522555079098369 *************************** 6. row *************************** primary_ts: 449522562802122752 secondary_ts: 449522562969632770 *************************** 7. row *************************** primary_ts: 449522570666442752 secondary_ts: 449522570833952770 *************************** 8. row *************************** primary_ts: 449522578530762752 secondary_ts: 449522578698534914 *************************** 9. row *************************** primary_ts: 449522586395082752 secondary_ts: 449522586588807169 skip invalid syncpoint primary_ts: 449522523480522753, first_ddl_ts: 449522527019466771 check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table processor_delay.t7 exists table processor_delay.t8 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/04 11:16:25.310 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/04 11:16:25.405 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark12000] [ts=449522587375763460] [2024/05/04 11:16:25.405 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark12000] [2024/05/04 11:16:25.405 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522587375763460] [2024/05/04 11:16:25.501 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:25.505 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:25.511 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:25.518 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ***************** properties ***************** "operationcount"="0" "updateproportion"="0" "threadcount"="10" "workload"="core" "requestdistribution"="uniform" "mysql.port"="4000" "recordcount"="50000" "fieldcount"="3" "readallfields"="true" "mysql.host"="127.0.0.1" "mysql.user"="root" "insertproportion"="0" "scanproportion"="0" "dotransactions"="false" "readproportion"="0" "mysql.db"="consistent_replicate_nfs" ********************************************** cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:25 CST 2024] <<<<<< run test case syncpoint success! >>>>>> table test.finish_mark not exists for 15-th check, retry later [2024/05/04 11:16:26.314 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark13000] [2024/05/04 11:16:26.314 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark13000] empty select id, val from test.availability1 where id=1 run task successfully test_owner_cleanup_stale_tasks pass wait process cdc.test exit for 1-th time... table processor_delay.t8 exists table processor_delay.t9 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... wait process cdc.test exit for 2-th time... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... table test.finish_mark not exists for 16-th check, retry later Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used check diff successfully wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_owner_retryable_error [Sat May 4 11:16:28 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/capture/capture-campaign-compacted-error=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.40724074.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:16:28.314 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522588280160262, "checkpoint_time": "2024-05-04 11:16:28.349", "error": null }' + echo '{ "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522588280160262, "checkpoint_time": "2024-05-04 11:16:28.349", "error": null }' { "upstream_id": 7364977875656973656, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522588280160262, "checkpoint_time": "2024-05-04 11:16:28.349", "error": null } ++ echo '{' '"upstream_id":' 7364977875656973656, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522588280160262, '"checkpoint_time":' '"2024-05-04' '11:16:28.349",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364977875656973656, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522588280160262, '"checkpoint_time":' '"2024-05-04' '11:16:28.349",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] table processor_delay.t9 exists table processor_delay.t10 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06698db40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:10229, start at 2024-05-04 11:16:28.187995317 +0800 CST m=+5.476001391 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:28.198 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:28.190 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:28.190 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06698db40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:10229, start at 2024-05-04 11:16:28.187995317 +0800 CST m=+5.476001391 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:28.198 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:28.190 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:28.190 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06698f340012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-9jh7r-sz6lx-x24r3, pid:10316, start at 2024-05-04 11:16:28.259282018 +0800 CST m=+5.494922276 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:28.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:28.237 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:28.237 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... [2024/05/04 11:16:29.714 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark14000] [2024/05/04 11:16:29.714 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark14000] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13094.out cli unsafe delete-service-gc-safepoint start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:30 CST 2024] <<<<<< run test case region_merge success! >>>>>> Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 17-th check, retry later [2024/05/04 11:16:30.327 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2024/05/04 11:16:30.404 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark13000] [ts=449522588309782533] [2024/05/04 11:16:30.404 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark13000] [2024/05/04 11:16:30.404 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522588309782533] [2024/05/04 11:16:30.407 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:30.410 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:30.414 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:30.419 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13127.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 table processor_delay.t10 exists table processor_delay.t11 not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11612.out cli tso query --pd=http://127.0.0.1:2379 reset and all metadata truncated in PD! PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.0% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_resume_with_checkpoint_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/04 11:16:31.715 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003ec7680 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522587671986177 MinTableBarrierTs:449522588982706191 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522587671986177, checkpointTs: 449522587671986177, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522587671986177, checkpointTs: 449522587671986177, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522587671986177, checkpointTs: 449522587671986177, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7697bfd4-3d99-4c69-b4ab-6d3a38202022 {"id":"7697bfd4-3d99-4c69-b4ab-6d3a38202022","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792588} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e332b4 7697bfd4-3d99-4c69-b4ab-6d3a38202022 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522587671986177,"min-table-barrier-ts":449522588982706191,"admin-job-type":0} /tidb/cdc/default/default/task/position/7697bfd4-3d99-4c69-b4ab-6d3a38202022/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003ec7680 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522587671986177 MinTableBarrierTs:449522588982706191 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522587671986177, checkpointTs: 449522587671986177, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522587671986177, checkpointTs: 449522587671986177, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522587671986177, checkpointTs: 449522587671986177, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7697bfd4-3d99-4c69-b4ab-6d3a38202022 {"id":"7697bfd4-3d99-4c69-b4ab-6d3a38202022","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792588} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e332b4 7697bfd4-3d99-4c69-b4ab-6d3a38202022 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522587671986177,"min-table-barrier-ts":449522588982706191,"admin-job-type":0} /tidb/cdc/default/default/task/position/7697bfd4-3d99-4c69-b4ab-6d3a38202022/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003ec7680 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522587671986177 MinTableBarrierTs:449522588982706191 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522587671986177, checkpointTs: 449522587671986177, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522587671986177, checkpointTs: 449522587671986177, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522587671986177, checkpointTs: 449522587671986177, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7697bfd4-3d99-4c69-b4ab-6d3a38202022 {"id":"7697bfd4-3d99-4c69-b4ab-6d3a38202022","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792588} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e332b4 7697bfd4-3d99-4c69-b4ab-6d3a38202022 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522587671986177,"min-table-barrier-ts":449522588982706191,"admin-job-type":0} /tidb/cdc/default/default/task/position/7697bfd4-3d99-4c69-b4ab-6d3a38202022/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table test.finish_mark not exists for 18-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [2024/05/04 11:16:33.009 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark15000] [2024/05/04 11:16:33.009 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark15000] "is-owner": true, run task successfully table processor_delay.t11 exists table processor_delay.t12 not exists for 1-th check, retry later + set +x + tso='449522589090709505 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522589090709505 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:16:33 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1164911651.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0669bc60000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01, pid:4468, start at 2024-05-04 11:16:31.14249155 +0800 CST m=+5.618989737 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:31.153 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:31.128 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:31.128 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0669bc60000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01, pid:4468, start at 2024-05-04 11:16:31.14249155 +0800 CST m=+5.618989737 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:31.153 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:31.128 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:31.128 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0669bdfc0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-7mp2h-b4mjm-fgx01, pid:4536, start at 2024-05-04 11:16:31.240198881 +0800 CST m=+5.637809833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:31.247 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:31.231 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:31.231 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/04 11:16:33.729 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/04 11:16:33.817 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark14000] [ts=449522589253763077] [2024/05/04 11:16:33.817 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark14000] [2024/05/04 11:16:33.817 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522589253763077] [2024/05/04 11:16:33.820 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:33.823 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/04 11:16:33.903 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:33.909 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table test.finish_mark not exists for 19-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... owner pid: 4077 owner id 7697bfd4-3d99-4c69-b4ab-6d3a38202022 [Sat May 4 11:16:34 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/owner-run-with-error=1*return(true);github.com/pingcap/tiflow/cdc/capture/capture-resign-failed=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.42054207.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:16:35.009 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] table processor_delay.t12 exists table processor_delay.t13 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0669d95c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242, pid:6795, start at 2024-05-04 11:16:32.989110007 +0800 CST m=+5.513243447 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:32.999 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:32.983 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:32.983 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0669d95c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242, pid:6795, start at 2024-05-04 11:16:32.989110007 +0800 CST m=+5.513243447 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:32.999 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:32.983 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:32.983 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0669dadc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-078bq-h57sb-9q242, pid:6863, start at 2024-05-04 11:16:33.085500703 +0800 CST m=+5.547743694 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:33.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:33.079 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:33.079 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } INSERT - Takes(s): 10.0, Count: 20935, OPS: 2091.2, Avg(us): 4727, Min(us): 689, Max(us): 883168, 95th(us): 4000, 99th(us): 87000 [Sat May 4 11:16:35 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.57915793.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: mysql... <<================= [Sat May 4 11:16:35 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> table test.finish_mark not exists for 20-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13244.out cli unsafe resolve-lock --region=62 PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:36 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e59f492-c5d8-464c-baf2-29b3c431ca83 {"id":"7e59f492-c5d8-464c-baf2-29b3c431ca83","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792593} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a383ed4 7e59f492-c5d8-464c-baf2-29b3c431ca83 /tidb/cdc/default/default/upstream/7364978040651666151 {"id":7364978040651666151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e59f492-c5d8-464c-baf2-29b3c431ca83 {"id":"7e59f492-c5d8-464c-baf2-29b3c431ca83","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792593} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a383ed4 7e59f492-c5d8-464c-baf2-29b3c431ca83 /tidb/cdc/default/default/upstream/7364978040651666151 {"id":7364978040651666151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e59f492-c5d8-464c-baf2-29b3c431ca83 {"id":"7e59f492-c5d8-464c-baf2-29b3c431ca83","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792593} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a383ed4 7e59f492-c5d8-464c-baf2-29b3c431ca83 /tidb/cdc/default/default/upstream/7364978040651666151 {"id":7364978040651666151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11705.out cli changefeed create --start-ts=449522589090709505 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: b25fe700-a779-4839-884c-d5359d9fda95 Info: {"upstream_id":7364978040651666151,"namespace":"default","id":"b25fe700-a779-4839-884c-d5359d9fda95","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:16:36.557631825+08:00","start_ts":449522589090709505,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522589090709505,"checkpoint_ts":449522589090709505,"checkpoint_time":"2024-05-04 11:16:31.441"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... coverage: 2.0% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 11:16:32.524 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 2 another is 6"] [2024/05/04 11:16:37.023 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/04 11:16:37.116 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark15000] [ts=449522590128275460] [2024/05/04 11:16:37.116 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark15000] [2024/05/04 11:16:37.116 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522590128275460] [2024/05/04 11:16:37.118 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:37.120 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark16000] [2024/05/04 11:16:37.120 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark16000] [2024/05/04 11:16:37.200 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:37.206 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:37.211 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024b6360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522590555832330 MinTableBarrierTs:449522590555832333 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7697bfd4-3d99-4c69-b4ab-6d3a38202022 {"id":"7697bfd4-3d99-4c69-b4ab-6d3a38202022","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792588} /tidb/cdc/default/__cdc_meta__/capture/f21b6c2a-5a39-4cb1-8d64-41da429c6787 {"id":"f21b6c2a-5a39-4cb1-8d64-41da429c6787","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792594} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e332b4 7697bfd4-3d99-4c69-b4ab-6d3a38202022 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33304 f21b6c2a-5a39-4cb1-8d64-41da429c6787 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522590555832330,"min-table-barrier-ts":449522590555832333,"admin-job-type":0} /tidb/cdc/default/default/task/position/7697bfd4-3d99-4c69-b4ab-6d3a38202022/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f21b6c2a-5a39-4cb1-8d64-41da429c6787/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024b6360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522590555832330 MinTableBarrierTs:449522590555832333 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7697bfd4-3d99-4c69-b4ab-6d3a38202022 {"id":"7697bfd4-3d99-4c69-b4ab-6d3a38202022","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792588} /tidb/cdc/default/__cdc_meta__/capture/f21b6c2a-5a39-4cb1-8d64-41da429c6787 {"id":"f21b6c2a-5a39-4cb1-8d64-41da429c6787","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792594} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e332b4 7697bfd4-3d99-4c69-b4ab-6d3a38202022 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33304 f21b6c2a-5a39-4cb1-8d64-41da429c6787 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522590555832330,"min-table-barrier-ts":449522590555832333,"admin-job-type":0} /tidb/cdc/default/default/task/position/7697bfd4-3d99-4c69-b4ab-6d3a38202022/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f21b6c2a-5a39-4cb1-8d64-41da429c6787/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024b6360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522590555832330 MinTableBarrierTs:449522590555832333 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7697bfd4-3d99-4c69-b4ab-6d3a38202022 {"id":"7697bfd4-3d99-4c69-b4ab-6d3a38202022","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792588} /tidb/cdc/default/__cdc_meta__/capture/f21b6c2a-5a39-4cb1-8d64-41da429c6787 {"id":"f21b6c2a-5a39-4cb1-8d64-41da429c6787","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792594} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e332b4 7697bfd4-3d99-4c69-b4ab-6d3a38202022 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33304 f21b6c2a-5a39-4cb1-8d64-41da429c6787 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522590555832330,"min-table-barrier-ts":449522590555832333,"admin-job-type":0} /tidb/cdc/default/default/task/position/7697bfd4-3d99-4c69-b4ab-6d3a38202022/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f21b6c2a-5a39-4cb1-8d64-41da429c6787/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "7697bfd4-3d99-4c69-b4ab-6d3a38202022" | grep id + set +x + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13275.out cli unsafe resolve-lock --region=62 --ts=449522589551558661 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 21-th check, retry later table resourcecontrol.finish_mark not exists for 1-th check, retry later table processor_delay.t14 exists table processor_delay.t15 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:38 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0993a547-0bbf-44e3-b390-f59fe389815c {"id":"0993a547-0bbf-44e3-b390-f59fe389815c","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792596} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a4b20ce 0993a547-0bbf-44e3-b390-f59fe389815c /tidb/cdc/default/default/upstream/7364978060752276385 {"id":7364978060752276385,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0993a547-0bbf-44e3-b390-f59fe389815c {"id":"0993a547-0bbf-44e3-b390-f59fe389815c","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792596} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a4b20ce 0993a547-0bbf-44e3-b390-f59fe389815c /tidb/cdc/default/default/upstream/7364978060752276385 {"id":7364978060752276385,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0993a547-0bbf-44e3-b390-f59fe389815c {"id":"0993a547-0bbf-44e3-b390-f59fe389815c","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792596} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a4b20ce 0993a547-0bbf-44e3-b390-f59fe389815c /tidb/cdc/default/default/upstream/7364978060752276385 {"id":7364978060752276385,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.5847.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 "cluster-id": "default" "id": "f21b6c2a-5a39-4cb1-8d64-41da429c6787", "cluster-id": "default" run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 11:16:39.120 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 27bb6994-899f-403b-92db-a9e07f8e5959 Info: {"upstream_id":7364978060752276385,"namespace":"default","id":"27bb6994-899f-403b-92db-a9e07f8e5959","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:16:39.542355487+08:00","start_ts":449522591158501378,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522591158501378,"checkpoint_ts":449522591158501378,"checkpoint_time":"2024-05-04 11:16:39.329"} PASS + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 158k 4583 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/04 11:16:40.003 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark17000] [2024/05/04 11:16:40.005 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark17000] table test.finish_mark not exists for 22-th check, retry later table resourcecontrol.finish_mark exists check diff successfully capture_id: f21b6c2a-5a39-4cb1-8d64-41da429c6787 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 35 100 35 0 0 28548 0 --:--:-- --:--:-- --:--:-- 35000 { "status": true, "message": "" }ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '7697bfd4-3d99-4c69-b4ab-6d3a38202022' 7697bfd4-3d99-4c69-b4ab-6d3a38202022 run task successfully ps -C cdc.test -o pid= | awk '{print $1}' | wc -l | grep 2 2 run task successfully test_owner_retryable_error pass + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8252.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [2024/05/04 11:16:41.199 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2024/05/04 11:16:41.215 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark16000] [ts=449522591176851462] [2024/05/04 11:16:41.215 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark16000] [2024/05/04 11:16:41.215 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522591176851462] [2024/05/04 11:16:41.225 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:41.230 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:41.303 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:41.308 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] wait process cdc.test exit for 2-th time... table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449522591408586753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522591408586753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:16:41 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.82858287.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:16:42 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> [2024/05/04 11:16:42.005 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] table test.finish_mark not exists for 23-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_gap_between_watch_capture [Sat May 4 11:16:42 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.43684370.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 199k 0 --:--:-- --:--:-- --:--:-- 177k { "version": "v8.2.0-alpha-54-gb297eae37", "git_hash": "b297eae37cdb06fa18bc5388b04d2564a81a0b87", "id": "19f52181-6c14-482b-a241-1ad6ae18638f", "pid": 12191, "is_owner": true ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) }wait process cdc.test exit for 1-th time... table processor_delay.t16 exists table processor_delay.t17 not exists for 1-th check, retry later [2024/05/04 11:16:43.602 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark18000] [2024/05/04 11:16:43.602 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark18000] start tidb cluster in /tmp/tidb_cdc_test/server_config_compatibility Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066a50b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt, pid:6478, start at 2024-05-04 11:16:40.656980559 +0800 CST m=+5.303619471 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:40.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:40.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:40.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066a50b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt, pid:6478, start at 2024-05-04 11:16:40.656980559 +0800 CST m=+5.303619471 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:40.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:40.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:40.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066a53c00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-p9h01-h8b43-b3hzt, pid:6555, start at 2024-05-04 11:16:40.820418755 +0800 CST m=+5.390965444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:40.827 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:40.816 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:40.816 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 24-th check, retry later [2024/05/04 11:16:44.099 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] [2024/05/04 11:16:44.200 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark17000] [ts=449522592146784261] [2024/05/04 11:16:44.200 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark17000] [2024/05/04 11:16:44.200 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522592146784261] [2024/05/04 11:16:44.204 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:44.208 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:44.213 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:44.218 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:16:44 CST 2024] <<<<<< run test case cli_with_auth success! >>>>>> table processor_resolved_ts_fallback.t1 exists [Sat May 4 11:16:44 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.59695971.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066a87a00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx, pid:6579, start at 2024-05-04 11:16:44.164550007 +0800 CST m=+5.141808942 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:44.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:44.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:44.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066a87a00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx, pid:6579, start at 2024-05-04 11:16:44.164550007 +0800 CST m=+5.141808942 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:44.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:44.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:44.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066a89d80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-zcsxg-0xclk-fk8rx, pid:6667, start at 2024-05-04 11:16:44.297975967 +0800 CST m=+5.178232524 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:44.303 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:44.278 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:44.278 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:44 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff4feec0-1989-404c-a04a-2a8bb2a0f605 {"id":"ff4feec0-1989-404c-a04a-2a8bb2a0f605","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a4ce8dc ff4feec0-1989-404c-a04a-2a8bb2a0f605 /tidb/cdc/default/default/upstream/7364978071844061199 {"id":7364978071844061199,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff4feec0-1989-404c-a04a-2a8bb2a0f605 {"id":"ff4feec0-1989-404c-a04a-2a8bb2a0f605","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a4ce8dc ff4feec0-1989-404c-a04a-2a8bb2a0f605 /tidb/cdc/default/default/upstream/7364978071844061199 {"id":7364978071844061199,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff4feec0-1989-404c-a04a-2a8bb2a0f605 {"id":"ff4feec0-1989-404c-a04a-2a8bb2a0f605","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a4ce8dc ff4feec0-1989-404c-a04a-2a8bb2a0f605 /tidb/cdc/default/default/upstream/7364978071844061199 {"id":7364978071844061199,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8341.out cli changefeed create --start-ts=449522591408586753 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: bdf00954-63a4-4be9-b4bd-9e9f17eed8f7 Info: {"upstream_id":7364978071844061199,"namespace":"default","id":"bdf00954-63a4-4be9-b4bd-9e9f17eed8f7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:16:45.259658571+08:00","start_ts":449522591408586753,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522591408586753,"checkpoint_ts":449522591408586753,"checkpoint_time":"2024-05-04 11:16:40.283"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) INSERT - Takes(s): 20.0, Count: 33693, OPS: 1685.1, Avg(us): 5830, Min(us): 689, Max(us): 883168, 95th(us): 54000, 99th(us): 87000 [2024/05/04 11:16:45.602 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] table test.finish_mark not exists for 25-th check, retry later Verifying downstream PD is started... [Sat May 4 11:16:45 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.79397941.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_changefeed_state http://127.0.0.1:2379 1029cc2a-7685-404e-b1b8-7948acb703af finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=1029cc2a-7685-404e-b1b8-7948acb703af + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 1029cc2a-7685-404e-b1b8-7948acb703af -s + info='{ "upstream_id": 7364977653118575242, "namespace": "default", "id": "1029cc2a-7685-404e-b1b8-7948acb703af", "state": "finished", "checkpoint_tso": 449522589680795652, "checkpoint_time": "2024-05-04 11:16:33.692", "error": null }' + echo '{ "upstream_id": 7364977653118575242, "namespace": "default", "id": "1029cc2a-7685-404e-b1b8-7948acb703af", "state": "finished", "checkpoint_tso": 449522589680795652, "checkpoint_time": "2024-05-04 11:16:33.692", "error": null }' { "upstream_id": 7364977653118575242, "namespace": "default", "id": "1029cc2a-7685-404e-b1b8-7948acb703af", "state": "finished", "checkpoint_tso": 449522589680795652, "checkpoint_time": "2024-05-04 11:16:33.692", "error": null } ++ echo '{' '"upstream_id":' 7364977653118575242, '"namespace":' '"default",' '"id":' '"1029cc2a-7685-404e-b1b8-7948acb703af",' '"state":' '"finished",' '"checkpoint_tso":' 449522589680795652, '"checkpoint_time":' '"2024-05-04' '11:16:33.692",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7364977653118575242, '"namespace":' '"default",' '"id":' '"1029cc2a-7685-404e-b1b8-7948acb703af",' '"state":' '"finished",' '"checkpoint_tso":' 449522589680795652, '"checkpoint_time":' '"2024-05-04' '11:16:33.692",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:41 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> + set +x go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 [2024/05/04 11:16:46.904 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark19000] [2024/05/04 11:16:46.904 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark19000] go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading golang.org/x/text v0.14.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/27bb6994-899f-403b-92db-a9e07f8e5959 {UpstreamID:7364978060752276385 Namespace:default ID:27bb6994-899f-403b-92db-a9e07f8e5959 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:16:39.542355487 +0800 CST StartTs:449522591158501378 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a27440 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522591210668034} {CheckpointTs:449522593084997633 MinTableBarrierTs:449522593084997633 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0993a547-0bbf-44e3-b390-f59fe389815c {"id":"0993a547-0bbf-44e3-b390-f59fe389815c","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792596} /tidb/cdc/default/__cdc_meta__/capture/4cebe879-9eea-46e6-a3d1-7a51ab03bada {"id":"4cebe879-9eea-46e6-a3d1-7a51ab03bada","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792604} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a4b20ce 0993a547-0bbf-44e3-b390-f59fe389815c /tidb/cdc/default/__cdc_meta__/owner/22318f419a4b2156 4cebe879-9eea-46e6-a3d1-7a51ab03bada /tidb/cdc/default/default/changefeed/info/27bb6994-899f-403b-92db-a9e07f8e5959 {"upstream-id":7364978060752276385,"namespace":"default","changefeed-id":"27bb6994-899f-403b-92db-a9e07f8e5959","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:16:39.542355487+08:00","start-ts":449522591158501378,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522591210668034} /tidb/cdc/default/default/changefeed/status/27bb6994-899f-403b-92db-a9e07f8e5959 {"checkpoint-ts":449522593084997633,"min-table-barrier-ts":449522593084997633,"admin-job-type":0} /tidb/cdc/default/default/task/position/0993a547-0bbf-44e3-b390-f59fe389815c/27bb6994-899f-403b-92db-a9e07f8e5959 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4cebe879-9eea-46e6-a3d1-7a51ab03bada/27bb6994-899f-403b-92db-a9e07f8e5959 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364978060752276385 {"id":7364978060752276385,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/27bb6994-899f-403b-92db-a9e07f8e5959 {UpstreamID:7364978060752276385 Namespace:default ID:27bb6994-899f-403b-92db-a9e07f8e5959 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:16:39.542355487 +0800 CST StartTs:449522591158501378 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a27440 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522591210668034} {CheckpointTs:449522593084997633 MinTableBarrierTs:449522593084997633 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0993a547-0bbf-44e3-b390-f59fe389815c {"id":"0993a547-0bbf-44e3-b390-f59fe389815c","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792596} /tidb/cdc/default/__cdc_meta__/capture/4cebe879-9eea-46e6-a3d1-7a51ab03bada {"id":"4cebe879-9eea-46e6-a3d1-7a51ab03bada","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792604} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a4b20ce 0993a547-0bbf-44e3-b390-f59fe389815c /tidb/cdc/default/__cdc_meta__/owner/22318f419a4b2156 4cebe879-9eea-46e6-a3d1-7a51ab03bada /tidb/cdc/default/default/changefeed/info/27bb6994-899f-403b-92db-a9e07f8e5959 {"upstream-id":7364978060752276385,"namespace":"default","changefeed-id":"27bb6994-899f-403b-92db-a9e07f8e5959","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:16:39.542355487+08:00","start-ts":449522591158501378,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522591210668034} /tidb/cdc/default/default/changefeed/status/27bb6994-899f-403b-92db-a9e07f8e5959 {"checkpoint-ts":449522593084997633,"min-table-barrier-ts":449522593084997633,"admin-job-type":0} /tidb/cdc/default/default/task/position/0993a547-0bbf-44e3-b390-f59fe389815c/27bb6994-899f-403b-92db-a9e07f8e5959 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4cebe879-9eea-46e6-a3d1-7a51ab03bada/27bb6994-899f-403b-92db-a9e07f8e5959 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364978060752276385 {"id":7364978060752276385,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/27bb6994-899f-403b-92db-a9e07f8e5959 {UpstreamID:7364978060752276385 Namespace:default ID:27bb6994-899f-403b-92db-a9e07f8e5959 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:16:39.542355487 +0800 CST StartTs:449522591158501378 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a27440 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522591210668034} {CheckpointTs:449522593084997633 MinTableBarrierTs:449522593084997633 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0993a547-0bbf-44e3-b390-f59fe389815c {"id":"0993a547-0bbf-44e3-b390-f59fe389815c","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792596} /tidb/cdc/default/__cdc_meta__/capture/4cebe879-9eea-46e6-a3d1-7a51ab03bada {"id":"4cebe879-9eea-46e6-a3d1-7a51ab03bada","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792604} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a4b20ce 0993a547-0bbf-44e3-b390-f59fe389815c /tidb/cdc/default/__cdc_meta__/owner/22318f419a4b2156 4cebe879-9eea-46e6-a3d1-7a51ab03bada /tidb/cdc/default/default/changefeed/info/27bb6994-899f-403b-92db-a9e07f8e5959 {"upstream-id":7364978060752276385,"namespace":"default","changefeed-id":"27bb6994-899f-403b-92db-a9e07f8e5959","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:16:39.542355487+08:00","start-ts":449522591158501378,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522591210668034} /tidb/cdc/default/default/changefeed/status/27bb6994-899f-403b-92db-a9e07f8e5959 {"checkpoint-ts":449522593084997633,"min-table-barrier-ts":449522593084997633,"admin-job-type":0} /tidb/cdc/default/default/task/position/0993a547-0bbf-44e3-b390-f59fe389815c/27bb6994-899f-403b-92db-a9e07f8e5959 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4cebe879-9eea-46e6-a3d1-7a51ab03bada/27bb6994-899f-403b-92db-a9e07f8e5959 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364978060752276385 {"id":7364978060752276385,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Sat May 4 11:16:47 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.80458047.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:16:47.617 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] [2024/05/04 11:16:47.804 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark18000] [ts=449522593054851078] [2024/05/04 11:16:47.804 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark18000] [2024/05/04 11:16:47.804 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522593054851078] [2024/05/04 11:16:47.808 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:47.812 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:47.816 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:47.903 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:48 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a5def0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522591342264330 MinTableBarrierTs:449522591342264335 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44380ec2-cad3-48ac-bd59-097b6dc4d1c3 {"id":"44380ec2-cad3-48ac-bd59-097b6dc4d1c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33364 44380ec2-cad3-48ac-bd59-097b6dc4d1c3 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522591342264330,"min-table-barrier-ts":449522591342264335,"admin-job-type":0} /tidb/cdc/default/default/task/position/44380ec2-cad3-48ac-bd59-097b6dc4d1c3/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a5def0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522591342264330 MinTableBarrierTs:449522591342264335 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44380ec2-cad3-48ac-bd59-097b6dc4d1c3 {"id":"44380ec2-cad3-48ac-bd59-097b6dc4d1c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33364 44380ec2-cad3-48ac-bd59-097b6dc4d1c3 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522591342264330,"min-table-barrier-ts":449522591342264335,"admin-job-type":0} /tidb/cdc/default/default/task/position/44380ec2-cad3-48ac-bd59-097b6dc4d1c3/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 + grep -q 'failed to get info:' {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a5def0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522591342264330 MinTableBarrierTs:449522591342264335 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44380ec2-cad3-48ac-bd59-097b6dc4d1c3 {"id":"44380ec2-cad3-48ac-bd59-097b6dc4d1c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33364 44380ec2-cad3-48ac-bd59-097b6dc4d1c3 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522591342264330,"min-table-barrier-ts":449522591342264335,"admin-job-type":0} /tidb/cdc/default/default/task/position/44380ec2-cad3-48ac-bd59-097b6dc4d1c3/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/prometheus/client_model v0.6.1 table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later [2024/05/04 11:16:48.904 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] table test.finish_mark not exists for 26-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.uber.org/mock v0.4.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:48 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a54a554-27b0-4697-9afc-54cd2100ae7f {"id":"9a54a554-27b0-4697-9afc-54cd2100ae7f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792606} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a67a0cd 9a54a554-27b0-4697-9afc-54cd2100ae7f /tidb/cdc/default/default/upstream/7364978099657963478 {"id":7364978099657963478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a54a554-27b0-4697-9afc-54cd2100ae7f {"id":"9a54a554-27b0-4697-9afc-54cd2100ae7f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792606} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a67a0cd 9a54a554-27b0-4697-9afc-54cd2100ae7f /tidb/cdc/default/default/upstream/7364978099657963478 {"id":7364978099657963478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a54a554-27b0-4697-9afc-54cd2100ae7f {"id":"9a54a554-27b0-4697-9afc-54cd2100ae7f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792606} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a67a0cd 9a54a554-27b0-4697-9afc-54cd2100ae7f /tidb/cdc/default/default/upstream/7364978099657963478 {"id":7364978099657963478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.7998.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-only-block-related-table Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7364978099657963478,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-04T11:16:49.276705283+08:00","start_ts":449522593738522627,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522593738522627,"checkpoint_ts":449522593738522627,"checkpoint_time":"2024-05-04 11:16:49.171"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 [2024/05/04 11:16:49.726 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20001] [2024/05/04 11:16:49.726 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/04 11:16:49.726 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20002] [2024/05/04 11:16:49.726 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/04 11:16:49.730 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20003] [2024/05/04 11:16:49.730 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/04 11:16:49.730 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20004] [2024/05/04 11:16:49.730 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 "is-owner": true, run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/05/04 11:16:50.302 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark20000] [2024/05/04 11:16:50.302 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20004] go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:50 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3cf44001-654f-478e-bb4d-823bbd31b759 {"id":"3cf44001-654f-478e-bb4d-823bbd31b759","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792607} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a7bfdc4 3cf44001-654f-478e-bb4d-823bbd31b759 /tidb/cdc/default/default/upstream/7364978117171480897 {"id":7364978117171480897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3cf44001-654f-478e-bb4d-823bbd31b759 {"id":"3cf44001-654f-478e-bb4d-823bbd31b759","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792607} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a7bfdc4 3cf44001-654f-478e-bb4d-823bbd31b759 /tidb/cdc/default/default/upstream/7364978117171480897 {"id":7364978117171480897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3cf44001-654f-478e-bb4d-823bbd31b759 {"id":"3cf44001-654f-478e-bb4d-823bbd31b759","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792607} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a7bfdc4 3cf44001-654f-478e-bb4d-823bbd31b759 /tidb/cdc/default/default/upstream/7364978117171480897 {"id":7364978117171480897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 + set +x go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 [2024/05/04 11:16:50.920 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] [2024/05/04 11:16:50.936 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark19000] [ts=449522593876934661] [2024/05/04 11:16:50.936 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark19000] [2024/05/04 11:16:50.936 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522593876934661] [2024/05/04 11:16:50.939 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:50.941 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:50.946 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:50.950 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:50.951 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark20000] table test.table1 not exists for 1-th check, retry later table test.finish_mark not exists for 27-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t20 exists table processor_delay.t21 not exists for 1-th check, retry later owner pid: 4373 owner id 44380ec2-cad3-48ac-bd59-097b6dc4d1c3 [Sat May 4 11:16:51 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.44964498.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // stage =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table test.table1 exists table test.table2 not exists for 1-th check, retry later table test.finish_mark not exists for 28-th check, retry later [Pipeline] // node table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully killing cdc1: 5796 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [2024/05/04 11:16:52.952 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark20000] [2024/05/04 11:16:52.967 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark20000] [ts=449522594509750277] [2024/05/04 11:16:52.967 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark20000] [2024/05/04 11:16:52.967 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449522594509750277] [2024/05/04 11:16:52.970 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:52.973 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:52.977 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 11:16:52.981 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 11:16:52.981 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/04 11:16:52.981 +08:00] [INFO] [case.go:615] ["bank test finished"] [valid=20] [tries=20] [ratio=1] + cleanup_process cdc.test [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 3-th time... table test.finish_mark not exists for 29-th check, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066b1f080033 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6, pid:6634, start at 2024-05-04 11:16:53.842708073 +0800 CST m=+26.923554144 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:53.851 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:53.826 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:53.826 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066b1f080033 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6, pid:6634, start at 2024-05-04 11:16:53.842708073 +0800 CST m=+26.923554144 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:53.851 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:53.826 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:53.826 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0669e9800003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-wfls9-dpq2f-872k6, pid:6710, start at 2024-05-04 11:16:34.018905536 +0800 CST m=+7.033894479 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:34.026 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:34.016 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:34.016 +0800 All versions after safe point can be accessed. (DO NOT EDIT) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ++ date + echo '[Sat May 4 11:16:55 CST 2024] <<<<<< run test case bank success! >>>>>>' [Sat May 4 11:16:55 CST 2024] <<<<<< run test case bank success! >>>>>> + stop_tidb_cluster go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025d6750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522595012280332 MinTableBarrierTs:449522595012280336 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44380ec2-cad3-48ac-bd59-097b6dc4d1c3 {"id":"44380ec2-cad3-48ac-bd59-097b6dc4d1c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792602} /tidb/cdc/default/__cdc_meta__/capture/a4ce9dd7-46fa-4473-b38a-184ebb02ec05 {"id":"a4ce9dd7-46fa-4473-b38a-184ebb02ec05","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33364 44380ec2-cad3-48ac-bd59-097b6dc4d1c3 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e333c0 a4ce9dd7-46fa-4473-b38a-184ebb02ec05 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522595012280332,"min-table-barrier-ts":449522595012280336,"admin-job-type":0} /tidb/cdc/default/default/task/position/44380ec2-cad3-48ac-bd59-097b6dc4d1c3/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a4ce9dd7-46fa-4473-b38a-184ebb02ec05/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025d6750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522595012280332 MinTableBarrierTs:449522595012280336 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44380ec2-cad3-48ac-bd59-097b6dc4d1c3 {"id":"44380ec2-cad3-48ac-bd59-097b6dc4d1c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792602} /tidb/cdc/default/__cdc_meta__/capture/a4ce9dd7-46fa-4473-b38a-184ebb02ec05 {"id":"a4ce9dd7-46fa-4473-b38a-184ebb02ec05","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33364 44380ec2-cad3-48ac-bd59-097b6dc4d1c3 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e333c0 a4ce9dd7-46fa-4473-b38a-184ebb02ec05 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522595012280332,"min-table-barrier-ts":449522595012280336,"admin-job-type":0} /tidb/cdc/default/default/task/position/44380ec2-cad3-48ac-bd59-097b6dc4d1c3/2264ab66-67c2-491a-8dc9-21a79802a34c {"chec+ grep -q 'failed to get info:' kpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a4ce9dd7-46fa-4473-b38a-184ebb02ec05/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025d6750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522595012280332 MinTableBarrierTs:449522595012280336 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44380ec2-cad3-48ac-bd59-097b6dc4d1c3 {"id":"44380ec2-cad3-48ac-bd59-097b6dc4d1c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792602} /tidb/cdc/default/__cdc_meta__/capture/a4ce9dd7-46fa-4473-b38a-184ebb02ec05 {"id":"a4ce9dd7-46fa-4473-b38a-184ebb02ec05","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e33364 44380ec2-cad3-48ac-bd59-097b6dc4d1c3 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e333c0 a4ce9dd7-46fa-4473-b38a-184ebb02ec05 /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522595012280332,"min-table-barrier-ts":449522595012280336,"admin-job-type":0} /tidb/cdc/default/default/task/position/44380ec2-cad3-48ac-bd59-097b6dc4d1c3/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a4ce9dd7-46fa-4473-b38a-184ebb02ec05/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "44380ec2-cad3-48ac-bd59-097b6dc4d1c3" | grep id Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later table test.table2 exists check diff failed 1-th time, retry later INSERT - Takes(s): 30.0, Count: 47262, OPS: 1575.7, Avg(us): 6265, Min(us): 689, Max(us): 883168, 95th(us): 75000, 99th(us): 88000 "id": "a4ce9dd7-46fa-4473-b38a-184ebb02ec05", "cluster-id": "default" "cluster-id": "default" run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066b29980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:10468, start at 2024-05-04 11:16:54.527175075 +0800 CST m=+5.284917092 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:54.536 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:54.502 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:54.502 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066b29980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:10468, start at 2024-05-04 11:16:54.527175075 +0800 CST m=+5.284917092 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:54.536 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:54.502 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:54.502 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066b2bd80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-l7q3d-gknk9-xs9pj, pid:10551, start at 2024-05-04 11:16:54.680466938 +0800 CST m=+5.387978611 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:54.688 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:54.646 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:54.646 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 30-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table processor_delay.t23 exists table processor_delay.t24 not exists for 1-th check, retry later check diff successfully [Sat May 4 11:16:57 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.2007920081.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later capture_id: a4ce9dd7-46fa-4473-b38a-184ebb02ec05 Run finished, takes 31.918625627s INSERT - Takes(s): 31.9, Count: 50000, OPS: 1566.7, Avg(us): 6303, Min(us): 689, Max(us): 883168, 95th(us): 75000, 99th(us): 88000 table consistent_replicate_nfs.usertable exists wait process cdc.test exit for 1-th time... Resume changefeed with checkpointTs3 449522596010000389 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.11928.out cli tso query --pd=http://127.0.0.1:2379 table test.table5 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t24 exists table processor_delay.t25 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... table test.finish_mark not exists for 31-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_only_block_related_table.finish_mark exists cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit wait process 7944 exit for 1-th time... + set +x + tso='449522596197433347 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522596197433347 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:17:00 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef47ee06-888c-4d54-b5eb-6ff8e2202ba7 {"id":"ef47ee06-888c-4d54-b5eb-6ff8e2202ba7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792617} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a46d605 ef47ee06-888c-4d54-b5eb-6ff8e2202ba7 /tidb/cdc/default/default/upstream/7364978067343326124 {"id":7364978067343326124,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef47ee06-888c-4d54-b5eb-6ff8e2202ba7 {"id":"ef47ee06-888c-4d54-b5eb-6ff8e2202ba7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792617} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a46d605 ef47ee06-888c-4d54-b5eb-6ff8e2202ba7 /tidb/cdc/default/default/upstream/7364978067343326124 {"id":7364978067343326124,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef47ee06-888c-4d54-b5eb-6ff8e2202ba7 {"id":"ef47ee06-888c-4d54-b5eb-6ff8e2202ba7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792617} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a46d605 ef47ee06-888c-4d54-b5eb-6ff8e2202ba7 /tidb/cdc/default/default/upstream/7364978067343326124 {"id":7364978067343326124,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.20362.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ wait process 7944 exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (7944) - No such process wait process 7944 exit for 3-th time... process 7944 already exit Create changefeed successfully! ID: 10a4110f-f832-485e-9789-95f9b7e1ae71 Info: {"upstream_id":7364978067343326124,"namespace":"default","id":"10a4110f-f832-485e-9789-95f9b7e1ae71","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:17:00.82052861+08:00","start_ts":449522596754489349,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522596754489349,"checkpoint_ts":449522596754489349,"checkpoint_time":"2024-05-04 11:17:00.676"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.table5 not exists for 2-th check, retry later [Sat May 4 11:17:00 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.80868088.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Sat May 4 11:17:01 CST 2024] <<<<<< START cdc server in server_config_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.1198411986.out server --log-file /tmp/tidb_cdc_test/server_config_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/server_config_compatibility/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/server_config_compatibility/conf/server.toml + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 32-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066b8cbc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2, pid:6574, start at 2024-05-04 11:17:00.85369327 +0800 CST m=+6.060279675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:00.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:00.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:00.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066b8cbc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2, pid:6574, start at 2024-05-04 11:17:00.85369327 +0800 CST m=+6.060279675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:00.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:00.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:00.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later + set +x nonempty select id, val from test.availability1 where id=1 and val=1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 33-th check, retry later run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.table5 exists table test.table6 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066ba3640011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-kk7r7-cnfh3-6svd2, pid:6655, start at 2024-05-04 11:17:02.310597646 +0800 CST m=+7.468130425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:02.317 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:02.297 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:02.297 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ***************** properties ***************** "recordcount"="1000" "mysql.user"="root" "mysql.db"="multi_rocks" "mysql.host"="127.0.0.1" "threadcount"="2" "mysql.port"="4000" "table"="a1" "insertproportion"="0" "readallfields"="true" "requestdistribution"="uniform" "workload"="core" "updateproportion"="0" "scanproportion"="0" "operationcount"="0" "readproportion"="0" "dotransactions"="false" ********************************************** table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later Run finished, takes 570.299275ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1796.7, Avg(us): 1110, Min(us): 846, Max(us): 13646, 95th(us): 2000, 99th(us): 2000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:17:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364978099657963478 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 11:16:49.276705283 +0800 CST StartTs:449522593738522627 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00398d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522593764737025} {CheckpointTs:449522597474074633 MinTableBarrierTs:449522597474074633 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28eebd89-24c5-47f2-be94-af6dd77beec9 {"id":"28eebd89-24c5-47f2-be94-af6dd77beec9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792621} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a67a1b0 28eebd89-24c5-47f2-be94-af6dd77beec9 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364978099657963478,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T11:16:49.276705283+08:00","start-ts":449522593738522627,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522593764737025} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449522597474074633,"min-table-barrier-ts":449522597474074633,"admin-job-type":0} /tidb/cdc/default/default/task/position/28eebd89-24c5-47f2-be94-af6dd77beec9/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364978099657963478 {"id":7364978099657963478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364978099657963478 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 11:16:49.276705283 +0800 CST StartTs:449522593738522627 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00398d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522593764737025} {CheckpointTs:449522597474074633 MinTableBarrierTs:449522597474074633 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28eebd89-24c5-47f2-be94-af6dd77beec9 {"id":"28eebd89-24c5-47f2-be94-af6dd77beec9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792621} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a67a1b0 28eebd89-24c5-47f2-be94-af6dd77beec9 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364978099657963478,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T11:16:49.276705283+08:00","start-ts":449522593738522627,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522593764737025} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449522597474074633,"min-table-barrier-ts":449522597474074633,"admin-job-type":0} /tidb/cdc/default/default/task/position/28eebd89-24c5-47f2-be94-af6dd77beec9/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364978099657963478 {"id":7364978099657963478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364978099657963478 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 11:16:49.276705283 +0800 CST StartTs:449522593738522627 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00398d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522593764737025} {CheckpointTs:449522597474074633 MinTableBarrierTs:449522597474074633 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522597474074634, checkpointTs: 449522597474074633, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28eebd89-24c5-47f2-be94-af6dd77beec9 {"id":"28eebd89-24c5-47f2-be94-af6dd77beec9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792621} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a67a1b0 28eebd89-24c5-47f2-be94-af6dd77beec9 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:17:04 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d5fccd07-963b-45ff-ac90-8892038aede8 {"id":"d5fccd07-963b-45ff-ac90-8892038aede8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792621} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a9dd7f3 d5fccd07-963b-45ff-ac90-8892038aede8 /tidb/cdc/default/default/upstream/7364978158169920826 {"id":7364978158169920826,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d5fccd07-963b-45ff-ac90-8892038aede8 {"upstream-id":7364978099657963478,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T11:16:49.276705283+08:00","start-ts":449522593738522627,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522593764737025} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table + grep -q 'etcd info' {"checkpoint-ts":449522597474074633,"min-table-barrier-ts":449522597474074633,"admin-job-type":0} {"id":"d5fccd07-963b-45ff-ac90-8892038aede8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792621} /tidb/cdc/default/default/task/position/28eebd89-24c5-47f2-be94-af6dd77beec9/ddl-only-block-related-table /tidb/cdc/default/__cdc_meta__/meta/meta-version {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 1 /tidb/cdc/default/default/upstream/7364978099657963478 /tidb/cdc/default/__cdc_meta__/owner/22318f419a9dd7f3 d5fccd07-963b-45ff-ac90-8892038aede8 /tidb/cdc/default/default/upstream/7364978158169920826 {"id":7364978099657963478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x {"id":7364978158169920826,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d5fccd07-963b-45ff-ac90-8892038aede8 {"id":"d5fccd07-963b-45ff-ac90-8892038aede8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792621} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a9dd7f3 d5fccd07-963b-45ff-ac90-8892038aede8 /tidb/cdc/default/default/upstream/7364978158169920826 {"id":7364978158169920826,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.12043.out cli changefeed create --start-ts=449522596197433347 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ ***************** properties ***************** "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.db"="multi_rocks" "scanproportion"="0" "operationcount"="0" "mysql.port"="4000" "requestdistribution"="uniform" "insertproportion"="0" "recordcount"="1000" "updateproportion"="0" "readproportion"="0" "mysql.user"="root" "threadcount"="2" "workload"="core" "table"="a2" "dotransactions"="false" ********************************************** ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 34-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.table6 exists check diff failed 1-th time, retry later check diff failed 1-th time, retry later Run finished, takes 562.112224ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1835.5, Avg(us): 1093, Min(us): 817, Max(us): 16876, 95th(us): 2000, 99th(us): 2000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "table"="a3" "dotransactions"="false" "operationcount"="0" "scanproportion"="0" "readproportion"="0" "updateproportion"="0" "recordcount"="1000" "mysql.port"="4000" "mysql.user"="root" "mysql.host"="127.0.0.1" "insertproportion"="0" "threadcount"="2" "requestdistribution"="uniform" "readallfields"="true" "workload"="core" "mysql.db"="multi_rocks" ********************************************** table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later Create changefeed successfully! ID: b2220f2f-ab1f-4908-a4c9-d012d925129b Info: {"upstream_id":7364978158169920826,"namespace":"default","id":"b2220f2f-ab1f-4908-a4c9-d012d925129b","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:17:05.596263445+08:00","start_ts":449522596197433347,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522596197433347,"checkpoint_ts":449522596197433347,"checkpoint_time":"2024-05-04 11:16:58.551"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later Run finished, takes 551.826754ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1863.7, Avg(us): 1075, Min(us): 779, Max(us): 15176, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "dotransactions"="false" "readproportion"="0" "operationcount"="0" "mysql.port"="4000" "updateproportion"="0" "workload"="core" "mysql.host"="127.0.0.1" "mysql.db"="multi_rocks" "recordcount"="1000" "readallfields"="true" "mysql.user"="root" "requestdistribution"="uniform" "table"="a4" "insertproportion"="0" "scanproportion"="0" "threadcount"="2" ********************************************** check diff failed 2-th time, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2654 0 --:--:-- --:--:-- --:--:-- 2662 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-04 11:16:53.495","puller_resolved_ts":"2024-05-04 11:16:46.495","last_synced_ts":"2024-05-04 11:14:37.845","now_ts":"2024-05-04 11:16:55.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '11:16:53.495","puller_resolved_ts":"2024-05-04' '11:16:46.495","last_synced_ts":"2024-05-04' '11:14:37.845","now_ts":"2024-05-04' '11:16:55.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 9086 7.7 0.0 13317516 143936 ? Sl 11:14 0:12 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 9149 5.1 0.0 13652944 137976 ? Sl 11:14 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714792620.8537993,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002a0e000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714792620.8538482,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714792620.9784327,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00331f500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714792620.9784958,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714792621.6469793,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d59dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714792621.6470375,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-04T11:17:05.413981+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efce00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:17:05.452369+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000945500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:17:05.505994+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fb81c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kill_owner_with_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Run finished, takes 569.615551ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1805.5, Avg(us): 1109, Min(us): 765, Max(us): 15645, 95th(us): 2000, 99th(us): 4000 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066be25c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:14304, start at 2024-05-04 11:17:06.359223634 +0800 CST m=+5.195539203 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:06.368 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:06.327 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:06.327 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066be25c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:14304, start at 2024-05-04 11:17:06.359223634 +0800 CST m=+5.195539203 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:06.368 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:06.327 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:06.327 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066be3c40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-q7cw3-fldzm-m9h1q, pid:14401, start at 2024-05-04 11:17:06.418352323 +0800 CST m=+5.180416212 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:06.424 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:06.417 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:06.417 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066bdb0c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg, pid:4071, start at 2024-05-04 11:17:05.906828845 +0800 CST m=+5.159969987 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:05.912 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:05.909 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:05.909 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066bdb0c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg, pid:4071, start at 2024-05-04 11:17:05.906828845 +0800 CST m=+5.159969987 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:05.912 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:05.909 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:05.909 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066bdd4c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1728-mdrtm-k4szs-dwlvg, pid:4164, start at 2024-05-04 11:17:06.030441522 +0800 CST m=+5.212958729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:06.038 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:06.003 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:06.003 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 35-th check, retry later empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later check diff successfully Resume changefeed with checkpointTs1 449522594135670790 table processor_delay.t28 exists table processor_delay.t29 not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.table3 not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.7937.out cli tso query --pd=http://127.0.0.1:2379 check diff successfully ***************** properties ***************** "workload"="core" "readallfields"="true" "insertproportion"="0" "requestdistribution"="uniform" "mysql.port"="4000" "operationcount"="0" "mysql.db"="multi_rocks" "dotransactions"="false" "scanproportion"="0" "readproportion"="0" "updateproportion"="0" "threadcount"="2" "mysql.user"="root" "table"="a5" "recordcount"="1000" "mysql.host"="127.0.0.1" ********************************************** wait process cdc.test exit for 1-th time... Run finished, takes 555.325409ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1857.7, Avg(us): 1076, Min(us): 821, Max(us): 16885, 95th(us): 2000, 99th(us): 2000 wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 36-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:17:09 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> check diff successfully check_ts_not_forward ddl-only-block-related-table table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15646.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449522598634323969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522598634323969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:17:09 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.79807982.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["consistent_replicate_nfs.usertable*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449522595533422679" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/04 11:17:09.860 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/04 11:17:09.861 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/04 11:17:09.861 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/04 11:17:09.861 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/04 11:17:09.861 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=449522592521650251] [resolvedTs=449522595533422679] [2024/05/04 11:17:09.862 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=327.31Β΅s] [2024/05/04 11:17:09.864 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 11:17:09.864 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 11:17:09.864 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 11:17:09.873 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 11:17:09.876 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/04 11:17:09.878 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/04 11:17:09.879 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 11:17:09.879 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 11:17:09.879 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 11:17:09.883 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 11:17:09.886 +08:00] [INFO] [mysql_ddl_sink.go:107] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/04 11:17:09.896 +08:00] [INFO] [file.go:284] ["ignore logs which commitTs is greater than resolvedTs"] [filename=127.0.0.1:8300_639fa55b-b603-430c-8dd0-d7b304b2e501_row_449522595559637041_55369d3c-b89b-462e-9e89-97016315cbee.log] [endTs=449522595533422679] [2024/05/04 11:17:09.902 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=40.187178ms] table test.table3 not exists for 2-th check, retry later [2024/05/04 11:17:07.310 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/05/04 11:17:07.310 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/05/04 11:17:07.406 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsba573e78_fb22_4cf0_9c9d_00279d7c3fb0"] [2024/05/04 11:17:07.481 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8b17b2f4_a430_45a8_befc_a0ea7b2aca34"] [2024/05/04 11:17:07.563 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse91f4805_a844_4405_bb9a_ae77d3c23613"] [2024/05/04 11:17:07.602 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9974fc96_7959_41c2_a747_eb0d20fae41e"] [2024/05/04 11:17:07.774 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd453790a_4f67_440b_bc4d_1ef9f1123950"] [2024/05/04 11:17:07.860 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsecf7bdcf_3bde_4c8d_a0c1_7fe2129df57d"] [2024/05/04 11:17:07.865 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd39acd88_577e_496e_b142_c3640b8d371f"] [2024/05/04 11:17:07.868 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4bee8dcc_c831_40d0_a604_dddb225019e4"] [2024/05/04 11:17:07.985 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:08.175 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:08.385 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:08.480 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:08.558 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:08.579 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/04 11:17:08.580 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:08.582 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/04 11:17:08.586 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:08.780 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:08.791 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:08.872 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:08.876 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:08.878 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:08.886 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:08.960 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:08.968 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:08.971 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:08.984 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:09.068 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/04 11:17:09.075 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/04 11:17:09.175 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:09.384 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:09.470 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:09.568 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:09.573 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:09.662 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/04 11:17:09.662 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:09.663 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/04 11:17:09.775 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:09.873 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:09.880 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:09.969 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:09.973 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:09.974 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:09.984 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:10.065 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:10.081 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:10.176 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:10.264 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table multi_rocks.finish_mark not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 11:17:10.477 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/04 11:17:10.560 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/04 11:17:10.569 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/04 11:17:10.574 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/04 11:17:10.583 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [Sat May 4 11:17:10 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.54705472.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:17:10.672 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:10.678 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:10.771 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:10.781 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/04 11:17:10.784 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:10.861 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/04 11:17:10.879 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:10.983 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:11.060 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:11.070 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:11.078 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:11.081 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:11.083 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table test.finish_mark not exists for 37-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/04 11:17:11.161 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:11.178 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:11.273 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] + set +x + tso='449522599140261890 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522599140261890 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:17:11 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1568415686.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:17:11.369 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:11.419 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}] [checkpointTs=449522595533422679] [2024/05/04 11:17:11.419 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}] [checkpointTs=449522595533422679] [2024/05/04 11:17:11.519 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=15484] [appliedDDLCount=0] [currentCheckpoint=449522595533422679] [2024/05/04 11:17:11.519 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/04 11:17:11.519 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/04 11:17:11.519 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/04 11:17:11.519 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/04 11:17:11.519 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/04 11:17:11.519 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/04 11:17:11.520 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/04 11:17:11.520 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/04 11:17:11.519 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/04 11:17:11.520 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/04 11:17:11.519 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/04 11:17:11.519 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/04 11:17:11.519 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/04 11:17:11.520 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/04 11:17:11.520 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/04 11:17:11.520 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] Apply redo log successfully check diff successfully [Sat May 4 11:17:11 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later [2024/05/04 11:17:11.684 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 11:17:11.759 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:11.775 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:11.782 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 11:17:11.864 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/04 11:17:11.869 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 11:17:11.873 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later table test.table3 not exists for 3-th check, retry later count(*) 47682 [2024/05/04 11:17:11.979 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:12.063 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/04 11:17:12.076 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/04 11:17:12.079 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] {"level":"warn","ts":"2024-05-04T11:17:11.414842+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efce00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:17:11.453218+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000945500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:17:11.506998+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fb81c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table multi_rocks.finish_mark not exists for 2-th check, retry later [2024/05/04 11:17:12.157 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:12.177 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:12.179 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 11:17:12.185 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 11:17:12.186 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:12.260 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:12.261 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] Verifying downstream PD is started... [2024/05/04 11:17:12.458 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 11:17:12.567 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:17:12 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/415a4d9e-7070-466a-bc47-b464b296cd0e {"id":"415a4d9e-7070-466a-bc47-b464b296cd0e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792629} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ab5a9da 415a4d9e-7070-466a-bc47-b464b296cd0e /tidb/cdc/default/default/upstream/7364978187952365645 {"id":7364978187952365645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/415a4d9e-7070-466a-bc47-b464b296cd0e {"id":"415a4d9e-7070-466a-bc47-b464b296cd0e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792629} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ab5a9da 415a4d9e-7070-466a-bc47-b464b296cd0e /tidb/cdc/default/default/upstream/7364978187952365645 {"id":7364978187952365645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/415a4d9e-7070-466a-bc47-b464b296cd0e {"id":"415a4d9e-7070-466a-bc47-b464b296cd0e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792629} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ab5a9da 415a4d9e-7070-466a-bc47-b464b296cd0e /tidb/cdc/default/default/upstream/7364978187952365645 {"id":7364978187952365645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 319e2068-035e-4341-ab2e-88f526a0cbda Info: {"upstream_id":7364978187952365645,"namespace":"default","id":"319e2068-035e-4341-ab2e-88f526a0cbda","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-04T11:17:12.588601964+08:00","start_ts":449522598634323969,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522598634323969,"checkpoint_ts":449522598634323969,"checkpoint_time":"2024-05-04 11:17:07.847"} [2024/05/04 11:17:12.883 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] table test.finish_mark not exists for 38-th check, retry later [2024/05/04 11:17:12.958 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 11:17:12.970 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 11:17:13.062 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 11:17:13.068 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/04 11:17:13.073 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/04 11:17:13.074 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 11:17:13.174 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 11:17:13.276 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 11:17:13.360 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 11:17:13.365 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 11:17:13.369 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 11:17:13.375 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 11:17:13.379 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 11:17:13.379 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 11:17:13.381 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/04 11:17:13.572 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] check data successfully table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:17:13 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12ad213c-a25b-4393-b4fc-a7ac50d06910 {"id":"12ad213c-a25b-4393-b4fc-a7ac50d06910","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792630} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419accf3cc 12ad213c-a25b-4393-b4fc-a7ac50d06910 /tidb/cdc/default/default/upstream/7364978213009177720 {"id":7364978213009177720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12ad213c-a25b-4393-b4fc-a7ac50d06910 {"id":"12ad213c-a25b-4393-b4fc-a7ac50d06910","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792630} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419accf3cc 12ad213c-a25b-4393-b4fc-a7ac50d06910 /tidb/cdc/default/default/upstream/7364978213009177720 {"id":7364978213009177720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12ad213c-a25b-4393-b4fc-a7ac50d06910 {"id":"12ad213c-a25b-4393-b4fc-a7ac50d06910","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792630} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419accf3cc 12ad213c-a25b-4393-b4fc-a7ac50d06910 /tidb/cdc/default/default/upstream/7364978213009177720 {"id":7364978213009177720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 3fee043a-0f4b-422e-bbca-5e9e11c3a478 Info: {"upstream_id":7364978213009177720,"namespace":"default","id":"3fee043a-0f4b-422e-bbca-5e9e11c3a478","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-04T11:17:13.49818266+08:00","start_ts":449522599240663041,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522599240663041,"checkpoint_ts":449522599240663041,"checkpoint_time":"2024-05-04 11:17:10.160"} table test.table3 not exists for 4-th check, retry later [2024/05/04 11:17:13.681 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/04 11:17:13.762 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/04 11:17:13.771 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/04 11:17:13.777 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/04 11:17:13.780 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] wait process cdc.test exit for 1-th time... table multi_rocks.finish_mark not exists for 3-th check, retry later [2024/05/04 11:17:14.067 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 11:17:14.174 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 11:17:14.258 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 11:17:14.260 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/04 11:17:14.263 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 11:17:14.267 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/04 11:17:14.268 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:17:14 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74f57fb4-2c9c-4ad9-b48f-dab656f79721 {"id":"74f57fb4-2c9c-4ad9-b48f-dab656f79721","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ad471d0 74f57fb4-2c9c-4ad9-b48f-dab656f79721 /tidb/cdc/default/default/upstream/7364978214344641812 {"id":7364978214344641812,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74f57fb4-2c9c-4ad9-b48f-dab656f79721 {"id":"74f57fb4-2c9c-4ad9-b48f-dab656f79721","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ad471d0 74f57fb4-2c9c-4ad9-b48f-dab656f79721 /tidb/cdc/default/default/upstream/7364978214344641812 {"id":7364978214344641812,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74f57fb4-2c9c-4ad9-b48f-dab656f79721 {"id":"74f57fb4-2c9c-4ad9-b48f-dab656f79721","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792631} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ad471d0 74f57fb4-2c9c-4ad9-b48f-dab656f79721 /tidb/cdc/default/default/upstream/7364978214344641812 {"id":7364978214344641812,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15736.out cli changefeed create --start-ts=449522599140261890 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ [2024/05/04 11:17:14.476 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 11:17:14.476 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 11:17:14.566 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 11:17:14.576 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 11:17:14.578 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 11:17:14.659 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 11:17:14.672 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 11:17:14.680 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 11:17:14.681 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table test.finish_mark not exists for 39-th check, retry later Create changefeed successfully! ID: 4734e71e-24b4-4c44-81c1-73ad38879858 Info: {"upstream_id":7364978214344641812,"namespace":"default","id":"4734e71e-24b4-4c44-81c1-73ad38879858","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:17:14.790358973+08:00","start_ts":449522599140261890,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522599140261890,"checkpoint_ts":449522599140261890,"checkpoint_time":"2024-05-04 11:17:09.777"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/04 11:17:14.775 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:17:14 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> table force_replicate_table.t0 not exists for 1-th check, retry later [2024/05/04 11:17:15.072 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 11:17:15.263 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 11:17:15.376 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/04 11:17:15.381 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 11:17:15.382 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 11:17:15.384 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/04 11:17:15.466 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 11:17:15.476 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/04 11:17:15.480 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 11:17:15.483 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/04 11:17:15.581 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 11:17:15.674 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 11:17:15.679 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later table test.table3 exists table test.table4 not exists for 1-th check, retry later [2024/05/04 11:17:15.763 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 11:17:15.767 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 11:17:15.875 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 11:17:15.961 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 11:17:15.965 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 11:17:15.968 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 11:17:15.970 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] table multi_rocks.finish_mark not exists for 4-th check, retry later + set +x [2024/05/04 11:17:16.370 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 11:17:16.468 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 11:17:16.664 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/04 11:17:16.671 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 11:17:16.676 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/04 11:17:16.677 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 11:17:16.778 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table savepoint.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 40-th check, retry later [2024/05/04 11:17:16.857 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 11:17:16.870 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 11:17:16.959 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 11:17:16.972 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 11:17:16.974 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 11:17:17.076 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 11:17:17.175 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 11:17:17.262 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 11:17:17.277 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 11:17:17.282 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/04 11:17:17.357 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 11:17:17.361 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/04 11:17:17.362 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/04 11:17:17.362 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 11:17:17.366 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/04 11:17:17.582 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0{"level":"warn","ts":"2024-05-04T11:17:17.416941+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efce00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:17:17.454975+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000945500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:17:17.507987+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fb81c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later table test.table4 exists table force_replicate_table.t0 exists table force_replicate_table.t1 not exists for 1-th check, retry later [2024/05/04 11:17:17.666 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 11:17:17.764 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/04 11:17:17.777 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 11:17:17.778 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/04 11:17:17.781 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] check diff failed 1-th time, retry later table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table multi_rocks.finish_mark not exists for 5-th check, retry later [2024/05/04 11:17:17.967 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 11:17:17.981 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 11:17:18.061 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 11:17:18.062 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 11:17:18.074 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 11:17:18.080 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 11:17:18.266 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 11:17:18.275 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] table test.finish_mark not exists for 3-th check, retry later [2024/05/04 11:17:18.478 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 11:17:18.573 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 11:17:18.580 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 11:17:18.581 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table savepoint.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 41-th check, retry later [2024/05/04 11:17:18.870 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 11:17:18.962 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 11:17:18.977 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/04 11:17:18.982 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/04 11:17:18.990 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/04 11:17:18.992 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 11:17:19.060 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 11:17:19.061 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/04 11:17:19.082 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 11:17:19.178 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 11:17:19.258 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 11:17:19.266 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 11:17:19.267 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 11:17:19.268 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 11:17:19.373 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 11:17:19.377 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table processor_delay.t34 exists table processor_delay.t35 not exists for 1-th check, retry later table force_replicate_table.t1 exists table force_replicate_table.t2 not exists for 1-th check, retry later [2024/05/04 11:17:19.679 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] empty select id, val from test.availability3 where id=1 run task successfully test_gap_between_watch_capture pass wait process cdc.test exit for 1-th time... [2024/05/04 11:17:19.780 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 11:17:19.860 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 11:17:19.862 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 11:17:20.072 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 11:17:20.075 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 11:17:20.076 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/04 11:17:20.165 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 11:17:20.172 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 11:17:20.172 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] table test.finish_mark not exists for 4-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... [2024/05/04 11:17:20.264 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 11:17:20.461 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table multi_rocks.finish_mark exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table savepoint.finish_mark exists [2024/05/04 11:17:20.473 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 11:17:20.562 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 11:17:20.570 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 11:17:20.659 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 11:17:20.663 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] wait process cdc.test exit for 1-th time... check diff successfully table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later [2024/05/04 11:17:20.759 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 11:17:20.864 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/04 11:17:20.868 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/04 11:17:20.872 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/04 11:17:20.877 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case delete_owner_key [Sat May 4 11:17:20 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.49124914.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:17:20.984 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 11:17:21.170 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 11:17:21.171 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 42-th check, retry later wait process cdc.test exit for 1-th time... [2024/05/04 11:17:21.262 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 11:17:21.371 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/04 11:17:21.374 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 11:17:21.381 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 11:17:21.460 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 11:17:21.460 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 11:17:21.463 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/04 11:17:21.473 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:17:21 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=1*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.88698871.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 11:17:21.657 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 11:17:21.665 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 11:17:21.676 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] wait process cdc.test exit for 3-th time... [2024/05/04 11:17:21.780 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 11:17:21.863 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 11:17:21.873 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 11:17:21.981 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] wait process cdc.test exit for 2-th time... table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later Aborted by Jenkins Admin Sending interrupt signal to process Killing processes cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:17:22 CST 2024] <<<<<< run test case multi_rocks success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:17:22 CST 2024] <<<<<< run test case savepoint success! >>>>>> kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 [2024/05/04 11:17:22.259 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 11:17:22.473 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 11:17:22.476 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 [2024/05/04 11:17:22.568 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 11:17:22.576 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/04 11:17:22.673 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/04 11:17:22.673 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/04 11:17:22.766 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 11:17:22.768 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/04 11:17:22.768 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 11:17:22.779 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] table common_1.recover_and_insert not exists for 2-th check, retry later table test.finish_mark not exists for 5-th check, retry later [2024/05/04 11:17:22.878 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 11:17:22.957 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 11:17:22.965 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 11:17:23.065 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 11:17:23.178 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0{"level":"warn","ts":"2024-05-04T11:17:23.417923+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efce00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:17:23.456224+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000945500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:17:23.509211+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fb81c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [2024/05/04 11:17:23.384 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 11:17:23.574 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 11:17:23.580 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 11:17:23.584 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] kill finished with exit code 0 Sending interrupt signal to process Killing processes [2024/05/04 11:17:23.659 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/04 11:17:23.759 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/04 11:17:23.759 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/04 11:17:23.873 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/04 11:17:23.877 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] table processor_delay.t36 exists table processor_delay.t37 not exists for 1-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes [2024/05/04 11:17:21.202 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 11.847234305s"] script returned exit code 143 \033[0;36m<<< Run all test success >>>\033[0m [2024/05/04 11:17:23.886 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/04 11:17:24.059 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/04 11:17:24.065 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/04 11:17:24.080 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/04 11:17:24.084 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:17:24 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031f90e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522602627563524 MinTableBarrierTs:449522602627563527 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522602627563527, checkpointTs: 449522602627563527, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522602627563524, checkpointTs: 449522602627563524, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522602627563524, checkpointTs: 449522602627563524, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e487130c-62e3-426c-9764-84d6f14a9cbd {"id":"e487130c-62e3-426c-9764-84d6f14a9cbd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792641} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e334e9 e487130c-62e3-426c-9764-84d6f14a9cbd /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522602627563524,"min-table-barrier-ts":449522602627563527,"admin-job-type":0} /tidb/cdc/default/default/task/position/e487130c-62e3-426c-9764-84d6f14a9cbd/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031f90e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522602627563524 MinTableBarrierTs:449522602627563527 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522602627563527, checkpointTs: 449522602627563527, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522602627563524, checkpointTs: 449522602627563524, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522602627563524, checkpointTs: 449522602627563524, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e487130c-62e3-426c-9764-84d6f14a9cbd {"id":"e487130c-62e3-426c-9764-84d6f14a9cbd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792641} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e334e9 e487130c-62e3-426c-9764-84d6f14a9cbd /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522602627563524,"min-table-barrier-ts":449522602627563527,"admin-job-type":0} /tidb/cdc/default/default/task/position/e487130c-62e3-426c-9764-84d6f14a9cbd/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2264ab66-67c2-491a-8dc9-21a79802a34c {UpstreamID:7364977674701950479 Namespace:default ID:2264ab66-67c2-491a-8dc9-21a79802a34c SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 11:15:11.267430246 +0800 CST StartTs:449522566307250177 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031f90e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522568063614982} {CheckpointTs:449522602627563524 MinTableBarrierTs:449522602627563527 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449522602627563527, checkpointTs: 449522602627563527, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522602627563524, checkpointTs: 449522602627563524, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522602627563524, checkpointTs: 449522602627563524, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e487130c-62e3-426c-9764-84d6f14a9cbd {"id":"e487130c-62e3-426c-9764-84d6f14a9cbd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714792641} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4198e334e9 e487130c-62e3-426c-9764-84d6f14a9cbd /tidb/cdc/default/default/changefeed/info/2264ab66-67c2-491a-8dc9-21a79802a34c {"upstream-id":7364977674701950479,"namespace":"default","changefeed-id":"2264ab66-67c2-491a-8dc9-21a79802a34c","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T11:15:11.267430246+08:00","start-ts":449522566307250177,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522568063614982} /tidb/cdc/default/default/changefeed/status/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":449522602627563524,"min-table-barrier-ts":449522602627563527,"admin-job-type":0} /tidb/cdc/default/default/task/position/e487130c-62e3-426c-9764-84d6f14a9cbd/2264ab66-67c2-491a-8dc9-21a79802a34c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977674701950479 {"id":7364977674701950479,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' {"level":"warn","ts":1714792644.0372078,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0021a4c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: read tcp 127.0.0.1:59660->127.0.0.1:2379: read: connection reset by peer"} script returned exit code 143 kill finished with exit code 1 Sending interrupt signal to process Killing processes [2024/05/04 11:17:24.171 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] kill finished with exit code 0 Sending interrupt signal to process Killing processes [2024/05/04 11:17:24.563 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 11:17:24.564 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 11:17:24.565 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/04 11:17:24.566 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 6-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 [2024/05/04 11:17:24.674 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/04 11:17:24.681 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/04 11:17:24.695 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] kill finished with exit code 0 Sending interrupt signal to process Killing processes [2024/05/04 11:17:24.992 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/04 11:17:25.076 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/04 11:17:25.094 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] {"level":"warn","ts":1714792645.1227105,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0023c2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 script returned exit code 143 [2024/05/04 11:17:25.170 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/04 11:17:25.364 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse46faf3a_6b26_4e92_ae1b_8fe3c44943ef"] [2024/05/04 11:17:25.410 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-05-04T11:17:25.405132+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efce00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-04T11:17:25.405183+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-04T11:17:25.44317+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000945500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-04T11:17:25.443205+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-04T11:17:25.49411+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fb81c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-04T11:17:25.494169+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} [2024/05/04 11:17:25.470 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6f6c9e10_b0c9_4a04_8ee4_2b1060157c6e"] [2024/05/04 11:17:25.478 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/04 11:17:25.486 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb52dcca7_e097_4f0d_b35f_fd54fd7bcabe"] [2024/05/04 11:17:25.490 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0acde921_bace_4c93_b82e_28df35f51f2b"] [2024/05/04 11:17:25.566 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/04 11:17:25.570 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/04 11:17:25.580 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsde707b61_95bc_43a8_ad76_1c17dc308f83"] [2024/05/04 11:17:25.669 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/04 11:17:25.960 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsbcdf1ad8_24de_4241_a13b_e3aa4be61299"] [2024/05/04 11:17:26.160 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs68c4edb0_655f_416e_998d_be3f6c253fea"] [2024/05/04 11:17:26.169 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:26.267 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:26.370 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] table test.finish_mark not exists for 7-th check, retry later [2024/05/04 11:17:26.567 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/04 11:17:26.860 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:27.071 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:27.072 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:27.161 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:27.173 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:27.259 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:27.463 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] kill finished with exit code 0 Sending interrupt signal to process Killing processes [2024/05/04 11:17:28.170 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:28.262 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:28.263 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] kill finished with exit code 0 Sending interrupt signal to process Killing processes table test.finish_mark not exists for 8-th check, retry later script returned exit code 143 [2024/05/04 11:17:28.465 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:28.564 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:28.664 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] table common_1.finish_mark not exists for 2-th check, retry later script returned exit code 143 [2024/05/04 11:17:28.757 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:28.769 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:28.871 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:28.967 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:29.181 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:29.260 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:29.275 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:29.278 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:29.364 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:29.382 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2024-05-04T11:17:29.419027+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efce00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:17:29.457491+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000945500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:17:29.510056+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fb81c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [2024/05/04 11:17:29.658 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:29.681 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:29.682 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/04 11:17:29.760 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 11:17:29.765 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] [2024/05/04 11:17:29.775 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:29.861 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsea5247c6_c92c_4f09_9644_3b3a3c014fe1"] [2024/05/04 11:17:29.878 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:29.882 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 11:17:29.967 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:30.063 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:30.071 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:30.275 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:30.361 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:30.375 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:30.559 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:30.662 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:30.766 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 11:17:30.774 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 11:17:30.778 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 11:17:30.864 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:30.880 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 11:17:30.961 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 11:17:30.962 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 11:17:31.060 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 11:17:31.072 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:31.072 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 11:17:31.078 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 11:17:31.273 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:31.360 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:31.476 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 11:17:31.477 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] kill finished with exit code 0 Sending interrupt signal to process Killing processes start tidb cluster in /tmp/tidb_cdc_test/changefeed_dup_error_restart Starting Upstream PD... [2024/05/04 11:17:31.672 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 11:17:31.768 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] kill finished with exit code 0 Sending interrupt signal to process Killing processes Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 11:17:31.860 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 11:17:31.860 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] script returned exit code 143 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... kill finished with exit code 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0Sending interrupt signal to process Killing processes ++ stop_tidb_cluster script returned exit code 143 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0kill finished with exit code 0 Sending interrupt signal to process Killing processes ++ stop_tidb_cluster script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes Verifying downstream PD is started... script returned exit code 143 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes Error: [CDC:ErrOwnerNotFound]owner not found {"level":"warn","ts":1714792655.7224872,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003aace00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Click here to forcibly terminate running steps Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) kill finished with exit code 0 Sending interrupt signal to process Killing processes Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) script returned exit code 143 kill finished with exit code 0 script returned exit code 143 {"level":"warn","ts":1714792655.452371,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003f20380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} {"level":"warn","ts":1714792655.4530916,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022b3340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} {"level":"warn","ts":1714792657.452393,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022b3340/127.0.0.1:2379","attempt":1,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} script returned exit code 143 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1728/tiflow-cdc already exists) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials {"level":"warn","ts":1714792651.6295068,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022308c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} {"level":"warn","ts":1714792653.630226,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022308c0/127.0.0.1:2379","attempt":1,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/resolve_lock/run.sh: line 1: 2836 Terminated GO111MODULE=on go run main.go -config ./config.toml 2>&1 2837 | tee $WORK_DIR/tester.log script returned exit code 143 [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G02' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G04' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G07' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G12' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G13' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G14' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G15' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G16' [Pipeline] } [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G18' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G19' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G20' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G21' [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G00' [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G05' [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: 0d5c9bfe-8715-473e-9e38-0e370ef77ece Failed in branch Matrix - TEST_GROUP = 'G02' Finished: ABORTED