Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j Still waiting to schedule task β€˜pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j’ is offline Agent pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-pw6fc-plpsn --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3151899aff80851d846cb2430696e34d6bb3d4e8" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-pw6fc" name: "pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=01eb17a8-cce3-42ee-aab6-4024f9b63239 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-pw6fc GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-ff467ea3dec646dcb34a434b6d5ddeaf65f47c579492826166e1422a0f0cf5b0 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-pw6fc pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build327764041=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-test-1731-pw6fc-plpsn-x3p0j bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 15426 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: de9c72fcfb6aae37 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Sat May 04 04:16:31 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/model/kv.go, cdc/model/sink.go, cdc/model/sink_test.go, cdc/processor/processor.go, cdc/processor/sinkmanager/manager.go, cdc/processor/sourcemanager/manager.go, cdc/redo/reader/reader.go, cdc/sink/dmlsink/factory/factory.go, cdc/sink/dmlsink/txn/mysql/mysql.go, cdc/sink/dmlsink/txn/mysql/mysql_test.go, cmd/kafka-consumer/main.go, cmd/pulsar-consumer/main.go, cmd/storage-consumer/main.go, errors.toml, pkg/applier/redo.go, pkg/applier/redo_test.go, pkg/errors/cdc_errors.go, pkg/errors/helper.go, tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml, tests/integration_tests/changefeed_dup_error_restart/conf/workload, tests/integration_tests/changefeed_dup_error_restart/run.sh, tests/integration_tests/force_replicate_table/run.sh, tests/integration_tests/open_protocol_handle_key_only/data/data.sql, tests/integration_tests/run_group.sh] [Pipeline] echo diff file not matched: cdc/model/kv.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534-20ef198) 203841536 bytes in 1.39 secs (146766702 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/.git/ .git HEAD is now at 20ef19816 check POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master = [up to date] refs/pull/10919/head -> origin/pr/10919/head Previous HEAD position was 20ef19816 check HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) 🚧 Checkouting to base SHA:be1553484fe4c03594eabb8d7435c694e5fd7224... HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) βœ… Checked. πŸŽ‰ 🧾 HEAD info: be1553484fe4c03594eabb8d7435c694e5fd7224 be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) 36e9e1bf6 cli(ticdc): allow client authentication to be enabled without tls (#11005) 🚧 Pre-merge heads of pull requests to base SHA: be1553484fe4c03594eabb8d7435c694e5fd7224 ... Updating be1553484..20ef19816 Fast-forward cdc/model/kv.go | 5 + cdc/model/sink.go | 35 ++- cdc/model/sink_test.go | 9 +- cdc/processor/processor.go | 21 +- cdc/processor/sinkmanager/manager.go | 5 + cdc/processor/sourcemanager/manager.go | 66 +++- cdc/redo/reader/reader.go | 21 +- cdc/sink/dmlsink/factory/factory.go | 8 +- cdc/sink/dmlsink/txn/mysql/mysql.go | 89 +++--- cdc/sink/dmlsink/txn/mysql/mysql_test.go | 2 +- cmd/kafka-consumer/main.go | 4 +- cmd/pulsar-consumer/main.go | 17 +- cmd/storage-consumer/main.go | 4 +- errors.toml | 5 + pkg/applier/redo.go | 303 +++++++++++++++++- pkg/applier/redo_test.go | 347 ++++++++++++++++++++- pkg/errors/cdc_errors.go | 4 + pkg/errors/helper.go | 19 ++ .../conf/diff_config.toml | 29 ++ .../changefeed_dup_error_restart/conf/workload | 13 + .../changefeed_dup_error_restart/run.sh | 54 ++++ .../integration_tests/force_replicate_table/run.sh | 4 +- .../open_protocol_handle_key_only/data/data.sql | 7 +- tests/integration_tests/run_group.sh | 5 +- 24 files changed, 973 insertions(+), 103 deletions(-) create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/workload create mode 100755 tests/integration_tests/changefeed_dup_error_restart/run.sh 🧾 Pre-merged result: 20ef198164f6ae8a5061b6b8ab1338f31dc54a4a 20ef19816 check d3434b194 f d5713c68f check βœ… Pre merged πŸŽ‰ βœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-be15534-20ef198 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 725 0 --:--:-- --:--:-- --:--:-- 732 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 839 0 --:--:-- --:--:-- --:--:-- 854 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1119 0 --:--:-- --:--:-- --:--:-- 1138 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 952 0 --:--:-- --:--:-- --:--:-- 976 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz 2024-05-04 12:16:56 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz [536570515/536570515] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz 2024-05-04 12:17:12 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz [187372022/187372022] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz 2024-05-04 12:17:46 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz [919098782/919098782] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz 2024-05-04 12:18:13 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz [456057803/456057803] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-04 12:18:19 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-04 12:18:21 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-04 12:18:21 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-04 12:18:22 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-04 12:18:25 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-04 12:18:41 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 12:18 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 12:18 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 12:18 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 12:18 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 12:18 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 12:18 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 12:18 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 12:18 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 12:18 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 12:18 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-57-g20ef19816" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 04:18:45" -X "github.com/pingcap/tiflow/pkg/version.GitHash=20ef198164f6ae8a5061b6b8ab1338f31dc54a4a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-57-g20ef19816"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/google/uuid v1.6.0 go: downloading golang.org/x/time v0.5.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading cloud.google.com/go v0.112.2 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading google.golang.org/api v0.170.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/jinzhu/now v1.1.5 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-57-g20ef19816" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 04:21:53" -X "github.com/pingcap/tiflow/pkg/version.GitHash=20ef198164f6ae8a5061b6b8ab1338f31dc54a4a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-57-g20ef19816"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-57-g20ef19816" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 04:22:00" -X "github.com/pingcap/tiflow/pkg/version.GitHash=20ef198164f6ae8a5061b6b8ab1338f31dc54a4a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-57-g20ef19816"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-57-g20ef19816" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 04:22:08" -X "github.com/pingcap/tiflow/pkg/version.GitHash=20ef198164f6ae8a5061b6b8ab1338f31dc54a4a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-57-g20ef19816"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-57-g20ef19816" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 04:22:08" -X "github.com/pingcap/tiflow/pkg/version.GitHash=20ef198164f6ae8a5061b6b8ab1338f31dc54a4a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-57-g20ef19816"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-57-g20ef19816" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 04:22:08" -X "github.com/pingcap/tiflow/pkg/version.GitHash=20ef198164f6ae8a5061b6b8ab1338f31dc54a4a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-57-g20ef19816"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-57-g20ef19816" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 04:22:08" -X "github.com/pingcap/tiflow/pkg/version.GitHash=20ef198164f6ae8a5061b6b8ab1338f31dc54a4a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-57-g20ef19816"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-57-g20ef19816" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 04:22:08" -X "github.com/pingcap/tiflow/pkg/version.GitHash=20ef198164f6ae8a5061b6b8ab1338f31dc54a4a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-57-g20ef19816"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-57-g20ef19816" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 04:22:08" -X "github.com/pingcap/tiflow/pkg/version.GitHash=20ef198164f6ae8a5061b6b8ab1338f31dc54a4a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-57-g20ef19816"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 12:27 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 12:18 .. -rwxr-xr-x. 1 jenkins jenkins 220M May 4 12:27 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 4 12:27 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 12:22 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 12:22 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 12:22 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M May 4 12:22 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-57-g20ef19816 Git Commit Hash: 20ef198164f6ae8a5061b6b8ab1338f31dc54a4a Git Branch: HEAD UTC Build Time: 2024-05-04 04:22:08 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache not saved (binary/pingcap/tiflow/cdc-integration-test/rev-be15534-20ef198 already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 12:27 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 12:18 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 12:27 bin -rwxr-xr-x. 1 jenkins jenkins 220M May 4 12:27 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 4 12:27 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 12:22 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 12:22 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 12:22 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 12:27 etc -rwxr-xr-x. 1 jenkins jenkins 17M May 4 12:27 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 12:27 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 12:27 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 12:27 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 4 12:27 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 4 12:27 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 4 12:27 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 4 12:27 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 4 12:27 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 4 12:27 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 4 12:27 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 4 12:27 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 4 12:27 minio -rwxr-xr-x. 1 jenkins jenkins 12M May 4 12:22 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 37M May 4 12:27 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 4 12:27 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 4 12:27 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 4 12:27 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 4 12:27 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 4 12:27 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M May 4 12:27 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 4 12:27 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 12:27 share -rwxr-xr-x. 1 jenkins jenkins 32M May 4 12:27 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M May 4 12:27 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 4 12:27 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 4 12:27 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M May 4 12:27 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 4 12:27 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 69.10 secs (53984010 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G18') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G19') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G20') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G21') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G18') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G19') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G20') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G21') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-2zc8x-wr1pl-g1qlp Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Agent pingcap-tiflow-pull-cdc-integration-test-1731-2zc8x-wr1pl-g1qlp is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-2zc8x-wr1pl --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "908063fb57848a2c83a92683f575644652e492e6" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-2zc8x" name: "pingcap-tiflow-pull-cdc-integration-test-1731-2zc8x-wr1pl-g1qlp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-2zc8x-wr1pl-g1qlp" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-2zc8x-wr1pl-g1qlp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-2zc8x-wr1pl-g1qlp in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87 The recommended git tool is: git Agent pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-cfl7x-khkls --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "21bdf5e5df5220a472e8a0518ec274339baf8c95" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-cfl7x" name: "pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Agent pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-lq24j-sdjs3 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "630d1e30cf6278f8426b18bc83f06c1014cd56ef" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-lq24j" name: "pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] { Running on pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb [Pipeline] checkout Agent pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-91kqp-8067w --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a42118fdf214220f420754ac86c689a118112c51" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-91kqp" name: "pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" The recommended git tool is: git Running on pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4acb4b; decorates RemoteLauncher[hudson.remoting.Channel@2c72c042:JNLP4-connect connection from 10.233.106.115/10.233.106.115:33970] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] checkout The recommended git tool is: git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-br2gk-nk79c --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ec0e7e17ed899258fcce31b18b2571634cdb69dd" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-br2gk" name: "pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@56be2ecb; decorates RemoteLauncher[hudson.remoting.Channel@3ccdb29a:JNLP4-connect connection from 10.233.127.85/10.233.127.85:51238] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2 Agent pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-l3kc8-cjn6b --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "da65c829369b2d1af71518d8aa356cd491616e91" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-l3kc8" name: "pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@20cf25b7; decorates RemoteLauncher[hudson.remoting.Channel@73366140:JNLP4-connect connection from 10.233.106.135/10.233.106.135:42484] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] checkout The recommended git tool is: git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@50566c06; decorates RemoteLauncher[hudson.remoting.Channel@78092414:JNLP4-connect connection from 10.233.126.221/10.233.126.221:34990] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1070f433; decorates RemoteLauncher[hudson.remoting.Channel@4bcbbb8b:JNLP4-connect connection from 10.233.72.174/10.233.72.174:36702] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@32c4aabb; decorates RemoteLauncher[hudson.remoting.Channel@7a08c029:JNLP4-connect connection from 10.233.126.48/10.233.126.48:38416] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-0df8h-dld9d --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c78ac18e6e1af00ad1307103dd0750ef65a8fa19" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-0df8h" name: "pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Agent pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-hmd9p-tvz3q --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e04f07ad34286e74cd0fa359859f7cd0107101e8" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-hmd9p" name: "pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w [Pipeline] node Running on pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch [Pipeline] withEnv Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] { [Pipeline] container [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7ffb1e01; decorates RemoteLauncher[hudson.remoting.Channel@51d67a2d:JNLP4-connect connection from 10.233.127.190/10.233.127.190:47864] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] stage [Pipeline] { (Test) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] stage [Pipeline] { (Test) [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-0q39v-0x5bp --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "325c7df2c13d6ba41f6044275ac3f7c8021e86a0" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-0q39v" name: "pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4d877589; decorates RemoteLauncher[hudson.remoting.Channel@72e93472:JNLP4-connect connection from 10.233.84.209/10.233.84.209:40760] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-d59k5-vscx3 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0b9e9d32fef076682e38e20298f175dac5adf10a" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-d59k5" name: "pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Running on pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-wmgf0-ls35n --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c21c977babf1dbec5156b4db1a3cd77b7514e622" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-wmgf0" name: "pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-gjgv5-1h70n --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5ac095a9d0c6993db2634b28af3d5bc40e00acb9" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-gjgv5" name: "pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-xjtcb-kc3v5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "babd4d63aadc218748eb654e0151d2c7ad958515" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-xjtcb" name: "pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 16.00 secs (233074800 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=fd56baa8-21c2-41d4-ac2d-2a31b1491ab8 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-cfl7x GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh pingcap_tiflow_pull_cdc_integration_test_1731-cfl7x GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] { The recommended git tool is: git [Pipeline] { [Pipeline] { [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w [Pipeline] stage Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms [Pipeline] { (Test) [Pipeline] stage Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0 [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] checkout The recommended git tool is: git [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@67ca9d7e; decorates RemoteLauncher[hudson.remoting.Channel@517515b1:JNLP4-connect connection from 10.233.126.23/10.233.126.23:42784] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] { Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@53c078c8; decorates RemoteLauncher[hudson.remoting.Channel@55865ea0:JNLP4-connect connection from 10.233.72.146/10.233.72.146:54130] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@585ed23f; decorates RemoteLauncher[hudson.remoting.Channel@11aef30c:JNLP4-connect connection from 10.233.69.191/10.233.69.191:55962] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1616de20; decorates RemoteLauncher[hudson.remoting.Channel@3904ba5c:JNLP4-connect connection from 10.233.71.132/10.233.71.132:51608] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1038bdca; decorates RemoteLauncher[hudson.remoting.Channel@19f249c:JNLP4-connect connection from 10.233.107.61/10.233.107.61:52700] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Agent pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-2nt2l-3g3s0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ede68b1cb9b9612d8aeb7f95d2cceb1b18ad3697" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-2nt2l" name: "pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-rgqf5-t2969 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a7853a429505a4205883e655e644191880920da8" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-rgqf5" name: "pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-t604k-7wvcn --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5b4929ab7188f3cce51d6f9524c1e8a0cb3527f3" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-t604k" name: "pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Running on pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-dm64w-5v4vw --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a488b7cd0deefbbd8268ae83b5ff1e9528565cfd" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-dm64w" name: "pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-x4lzj-z4m7f --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "82837c97c9333011c2ccfb8d0fce476ad44bea40" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-x4lzj" name: "pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Avoid second fetch Running on pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-wtlx8-mmqds Agent pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-j6n6k-019xs --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0fee9ae7f83a62415849e3e56c3faa1b74020136" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-j6n6k" name: "pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "697b454d049207ed850f39ae20318af303dfea93" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-wtlx8" name: "pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Running on pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0 is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-9570k-8qxk6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b1260dc286ee48837760b15855fd0ec5e06f773b" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-9570k" name: "pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Running on pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test Agent pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms is provisioned from template pingcap_tiflow_pull_cdc_integration_test_1731-bxqzg-nvfw0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2a8b8d5b2d89ed74e73b4909fb8c5672d7b2ca02" jenkins/label: "pingcap_tiflow_pull_cdc_integration_test_1731-bxqzg" name: "pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 7.12 secs (523857429 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=52684899-baac-451a-af52-dbfee1dd0cf6 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-lq24j GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t pingcap_tiflow_pull_cdc_integration_test_1731-lq24j GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07750fc40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:1272, start at 2024-05-04 12:29:30.996261011 +0800 CST m=+5.449956105 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:31:31.003 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:29:30.993 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:19:30.993 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07750fc40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:1272, start at 2024-05-04 12:29:30.996261011 +0800 CST m=+5.449956105 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:31:31.003 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:29:30.993 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:19:30.993 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077510780014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:1344, start at 2024-05-04 12:29:31.063963867 +0800 CST m=+5.430542215 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:31:31.070 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:29:31.038 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:19:31.038 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2600.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='449523738527137793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523738527137793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:29:37 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.26342636.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:29:40 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38b7f317-da4d-4daf-836d-272845134d93 {"id":"38b7f317-da4d-4daf-836d-272845134d93","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714796978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dd1ca9d4 38b7f317-da4d-4daf-836d-272845134d93 /tidb/cdc/default/default/upstream/7364996872504676251 {"id":7364996872504676251,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38b7f317-da4d-4daf-836d-272845134d93 {"id":"38b7f317-da4d-4daf-836d-272845134d93","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714796978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dd1ca9d4 38b7f317-da4d-4daf-836d-272845134d93 /tidb/cdc/default/default/upstream/7364996872504676251 {"id":7364996872504676251,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38b7f317-da4d-4daf-836d-272845134d93 {"id":"38b7f317-da4d-4daf-836d-272845134d93","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714796978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dd1ca9d4 38b7f317-da4d-4daf-836d-272845134d93 /tidb/cdc/default/default/upstream/7364996872504676251 {"id":7364996872504676251,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2691.out cli changefeed create --start-ts=449523738527137793 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 0a9b616d-7fac-42aa-8c85-2aab56ad64f2 Info: {"upstream_id":7364996872504676251,"namespace":"default","id":"0a9b616d-7fac-42aa-8c85-2aab56ad64f2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:29:41.21467065+08:00","start_ts":449523738527137793,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523738527137793,"checkpoint_ts":449523738527137793,"checkpoint_time":"2024-05-04 12:29:36.193"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 11.94 secs (312551885 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0775e6d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:1236, start at 2024-05-04 12:29:44.780810302 +0800 CST m=+5.393152970 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:31:44.788 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:29:44.757 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:19:44.757 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0775e6d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:1236, start at 2024-05-04 12:29:44.780810302 +0800 CST m=+5.393152970 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:31:44.788 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:29:44.757 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:19:44.757 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0775e6cc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:1322, start at 2024-05-04 12:29:44.801075404 +0800 CST m=+5.357089052 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:31:44.809 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:29:44.805 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:19:44.805 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2642.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449523742161502209 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523742161502209 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:29:51 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.26902692.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 7.42 secs (503075018 bytes/sec) [Pipeline] { [Pipeline] cache table row_format.finish_mark not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:29:54 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64e3fe96-8e4c-4561-ad34-15f04068a018 {"id":"64e3fe96-8e4c-4561-ad34-15f04068a018","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714796991} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dd50abd7 64e3fe96-8e4c-4561-ad34-15f04068a018 /tidb/cdc/default/default/upstream/7364996929222236562 {"id":7364996929222236562,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64e3fe96-8e4c-4561-ad34-15f04068a018 {"id":"64e3fe96-8e4c-4561-ad34-15f04068a018","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714796991} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dd50abd7 64e3fe96-8e4c-4561-ad34-15f04068a018 /tidb/cdc/default/default/upstream/7364996929222236562 {"id":7364996929222236562,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64e3fe96-8e4c-4561-ad34-15f04068a018 {"id":"64e3fe96-8e4c-4561-ad34-15f04068a018","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714796991} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dd50abd7 64e3fe96-8e4c-4561-ad34-15f04068a018 /tidb/cdc/default/default/upstream/7364996929222236562 {"id":7364996929222236562,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2746.out cli changefeed create --start-ts=449523742161502209 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: b9146078-376d-4e10-8d50-12905c7764fc Info: {"upstream_id":7364996929222236562,"namespace":"default","id":"b9146078-376d-4e10-8d50-12905c7764fc","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:29:55.050339758+08:00","start_ts":449523742161502209,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523742161502209,"checkpoint_ts":449523742161502209,"checkpoint_time":"2024-05-04 12:29:50.057"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table row_format.finish_mark not exists for 3-th check, retry later + set +x table row_format.finish_mark not exists for 4-th check, retry later table foreign_key.finish_mark not exists for 1-th check, retry later table row_format.finish_mark not exists for 5-th check, retry later table foreign_key.finish_mark not exists for 2-th check, retry later table row_format.finish_mark not exists for 6-th check, retry later table foreign_key.finish_mark not exists for 3-th check, retry later table row_format.finish_mark not exists for 7-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 9.45 secs (394651593 bytes/sec) [Pipeline] { [Pipeline] cache table foreign_key.finish_mark not exists for 4-th check, retry later table row_format.finish_mark not exists for 8-th check, retry later table foreign_key.finish_mark not exists for 5-th check, retry later table row_format.finish_mark not exists for 9-th check, retry later table foreign_key.finish_mark not exists for 6-th check, retry later table row_format.finish_mark not exists for 10-th check, retry later table foreign_key.finish_mark not exists for 7-th check, retry later table row_format.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table foreign_key.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:30:14 CST 2024] <<<<<< run test case row_format success! >>>>>> table foreign_key.finish_mark not exists for 9-th check, retry later table foreign_key.finish_mark not exists for 10-th check, retry later table foreign_key.finish_mark not exists for 11-th check, retry later table foreign_key.finish_mark not exists for 12-th check, retry later table foreign_key.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:30:24 CST 2024] <<<<<< run test case foreign_key success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 22.96 secs (162438206 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077935540014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:3898, start at 2024-05-04 12:30:38.970228827 +0800 CST m=+5.508178030 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:32:38.977 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:30:38.983 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:20:38.983 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077935540014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:3898, start at 2024-05-04 12:30:38.970228827 +0800 CST m=+5.508178030 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:32:38.977 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:30:38.983 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:20:38.983 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077936dc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:3961, start at 2024-05-04 12:30:39.05449023 +0800 CST m=+5.517838280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:32:39.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:30:39.031 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:20:39.031 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5201.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449523756337201153 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523756337201153 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:30:45 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.52355237.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 14.20 secs (262651017 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility changefeed_dup_error_restart kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=be436aee-0f3f-4c37-98c0-39696a85eea4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-91kqp GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx pingcap_tiflow_pull_cdc_integration_test_1731-91kqp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=4a7122bb-4179-43fe-af84-cb037961af33 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-br2gk GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq pingcap_tiflow_pull_cdc_integration_test_1731-br2gk GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=ec35a0c6-8cdf-47d1-ab4a-607698cfbdba BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-l3kc8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-l3kc8 pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] cache + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G01 Run cases: open_protocol_handle_key_only PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=6f29829a-a8d3-4403-b80e-b5640a522f84 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-2zc8x-wr1pl-g1qlp HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-2zc8x GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-2zc8x pingcap-tiflow-pull-cdc-integration-test-1731-2zc8x-wr1pl-g1qlp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-2zc8x-wr1pl-g1qlp GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Sat May 4 12:30:49 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:30:48 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc401941-5129-4059-adb7-a171a53bc0d6 {"id":"cc401941-5129-4059-adb7-a171a53bc0d6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797046} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de1fffd7 cc401941-5129-4059-adb7-a171a53bc0d6 /tidb/cdc/default/default/upstream/7364997157130930604 {"id":7364997157130930604,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc401941-5129-4059-adb7-a171a53bc0d6 {"id":"cc401941-5129-4059-adb7-a171a53bc0d6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797046} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de1fffd7 cc401941-5129-4059-adb7-a171a53bc0d6 /tidb/cdc/default/default/upstream/7364997157130930604 {"id":7364997157130930604,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc401941-5129-4059-adb7-a171a53bc0d6 {"id":"cc401941-5129-4059-adb7-a171a53bc0d6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797046} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de1fffd7 cc401941-5129-4059-adb7-a171a53bc0d6 /tidb/cdc/default/default/upstream/7364997157130930604 {"id":7364997157130930604,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5298.out cli changefeed create --start-ts=449523756337201153 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=28c68a17-4f43-49b4-9954-a52daf6d4f8d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-0df8h GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87 pingcap_tiflow_pull_cdc_integration_test_1731-0df8h GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Create changefeed successfully! ID: d6e80ad0-710e-4e4a-924b-e0044eba813c Info: {"upstream_id":7364997157130930604,"namespace":"default","id":"d6e80ad0-710e-4e4a-924b-e0044eba813c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:30:49.246555855+08:00","start_ts":449523756337201153,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523756337201153,"checkpoint_ts":449523756337201153,"checkpoint_time":"2024-05-04 12:30:44.133"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0779f29c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:3892, start at 2024-05-04 12:30:51.072805552 +0800 CST m=+5.333539260 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:32:51.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:30:51.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:20:51.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0779f29c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:3892, start at 2024-05-04 12:30:51.072805552 +0800 CST m=+5.333539260 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:32:51.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:30:51.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:20:51.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0779f3540003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:3981, start at 2024-05-04 12:30:51.095708932 +0800 CST m=+5.278451107 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:32:51.103 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:30:51.093 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:20:51.093 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... table drop_tables.c not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5223.out cli tso query --pd=http://127.0.0.1:2379 \033[0;36m<<< Run all test success >>>\033[0m Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449523759276883969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523759276883969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 12:30:57 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.52775279.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 9.90 secs (376908905 bytes/sec) [Pipeline] { [Pipeline] sh table drop_tables.c not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:31:00 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e0cfab2-2ea0-4cad-b20d-82ddbd8bd948 {"id":"8e0cfab2-2ea0-4cad-b20d-82ddbd8bd948","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de55f4f3 8e0cfab2-2ea0-4cad-b20d-82ddbd8bd948 /tidb/cdc/default/default/upstream/7364997218287149300 {"id":7364997218287149300,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e0cfab2-2ea0-4cad-b20d-82ddbd8bd948 {"id":"8e0cfab2-2ea0-4cad-b20d-82ddbd8bd948","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de55f4f3 8e0cfab2-2ea0-4cad-b20d-82ddbd8bd948 /tidb/cdc/default/default/upstream/7364997218287149300 {"id":7364997218287149300,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e0cfab2-2ea0-4cad-b20d-82ddbd8bd948 {"id":"8e0cfab2-2ea0-4cad-b20d-82ddbd8bd948","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de55f4f3 8e0cfab2-2ea0-4cad-b20d-82ddbd8bd948 /tidb/cdc/default/default/upstream/7364997218287149300 {"id":7364997218287149300,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5337.out cli changefeed create --start-ts=449523759276883969 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=6482f206-fb10-4800-93ca-5e7b815160b0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-hmd9p GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg pingcap_tiflow_pull_cdc_integration_test_1731-hmd9p GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] { Create changefeed successfully! ID: dda1f159-3572-4681-8bc0-3bd90bedbcc3 Info: {"upstream_id":7364997218287149300,"namespace":"default","id":"dda1f159-3572-4681-8bc0-3bd90bedbcc3","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:31:01.418047399+08:00","start_ts":449523759276883969,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523759276883969,"checkpoint_ts":449523759276883969,"checkpoint_time":"2024-05-04 12:30:55.347"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] stage The recommended git tool is: git [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@30e4c020; decorates RemoteLauncher[hudson.remoting.Channel@5ffd6465:JNLP4-connect connection from 10.233.97.159/10.233.97.159:57828] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + set +x table drop_tables.c exists check diff successfully Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@43dd9475; decorates RemoteLauncher[hudson.remoting.Channel@75d853f7:JNLP4-connect connection from 10.233.71.110/10.233.71.110:51044] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@31a89383; decorates RemoteLauncher[hudson.remoting.Channel@3e098df4:JNLP4-connect connection from 10.233.126.239/10.233.126.239:34758] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077a9a44001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq, pid:1236, start at 2024-05-04 12:31:01.823402521 +0800 CST m=+5.206673505 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:01.830 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:01.828 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:01.828 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077a9a44001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq, pid:1236, start at 2024-05-04 12:31:01.823402521 +0800 CST m=+5.206673505 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:01.830 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:01.828 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:01.828 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077a9bb00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq, pid:1313, start at 2024-05-04 12:31:01.881675531 +0800 CST m=+5.204657345 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:01.887 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:01.868 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:01.868 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1c564925; decorates RemoteLauncher[hudson.remoting.Channel@10b5187f:JNLP4-connect connection from 10.233.68.129/10.233.68.129:45192] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5fcee4af; decorates RemoteLauncher[hudson.remoting.Channel@1d40f82:JNLP4-connect connection from 10.233.88.181/10.233.88.181:56780] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@25e90ba4; decorates RemoteLauncher[hudson.remoting.Channel@61cbdd57:JNLP4-connect connection from 10.233.69.228/10.233.69.228:53050] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@71476de5; decorates RemoteLauncher[hudson.remoting.Channel@700255c2:JNLP4-connect connection from 10.233.108.245/10.233.108.245:34878] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3cf29340; decorates RemoteLauncher[hudson.remoting.Channel@76aedf3a:JNLP4-connect connection from 10.233.93.165/10.233.93.165:47846] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4504fdf7; decorates RemoteLauncher[hudson.remoting.Channel@6660f8d0:JNLP4-connect connection from 10.233.72.94/10.233.72.94:54816] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git wait process cdc.test exit for 1-th time... Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077aaf640003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx, pid:1196, start at 2024-05-04 12:31:03.133665683 +0800 CST m=+5.512725217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:03.142 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:03.129 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:03.129 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077a9da80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:1235, start at 2024-05-04 12:31:02.023654786 +0800 CST m=+5.336525904 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:02.031 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:01.994 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:01.994 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077a9da80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:1235, start at 2024-05-04 12:31:02.023654786 +0800 CST m=+5.336525904 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:02.031 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:01.994 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:01.994 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077aa0c80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:1307, start at 2024-05-04 12:31:02.233194048 +0800 CST m=+5.494715216 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:02.242 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:02.244 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:02.244 +0800 All versions after safe point can be accessed. (DO NOT EDIT) start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sat May 4 12:31:05 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.26762678.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077abbdc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87, pid:1212, start at 2024-05-04 12:31:03.949647137 +0800 CST m=+5.492153091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:03.956 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:03.927 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:03.927 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:31:05 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077aaf640003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx, pid:1196, start at 2024-05-04 12:31:03.133665683 +0800 CST m=+5.512725217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:03.142 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:03.129 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:03.129 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077ab0180006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx, pid:1266, start at 2024-05-04 12:31:03.182487131 +0800 CST m=+5.496003018 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:03.190 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:03.174 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:03.174 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc already exists) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Avoid second fetch Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] withCredentials > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077abbdc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87, pid:1212, start at 2024-05-04 12:31:03.949647137 +0800 CST m=+5.492153091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:03.956 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:03.927 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:03.927 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077abd580014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87, pid:1292, start at 2024-05-04 12:31:04.054763354 +0800 CST m=+5.514440876 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:04.062 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:04.022 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:04.022 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // cache [Pipeline] } [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Sat May 4 12:31:07 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.26072609.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Sat May 4 12:31:07 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> [Pipeline] { [Pipeline] // dir [Pipeline] } [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.25842586.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] cache Commit message: "fix(br): use failpoint tidb-server instead (#2951)" + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:31:08 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5426a961-a6aa-4f7d-a877-94503a62c6f8 {"id":"5426a961-a6aa-4f7d-a877-94503a62c6f8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797065} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de8077c8 5426a961-a6aa-4f7d-a877-94503a62c6f8 /tidb/cdc/default/default/upstream/7364997265750694396 {"id":7364997265750694396,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5426a961-a6aa-4f7d-a877-94503a62c6f8 {"id":"5426a961-a6aa-4f7d-a877-94503a62c6f8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797065} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de8077c8 5426a961-a6aa-4f7d-a877-94503a62c6f8 /tidb/cdc/default/default/upstream/7364997265750694396 {"id":7364997265750694396,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5426a961-a6aa-4f7d-a877-94503a62c6f8 {"id":"5426a961-a6aa-4f7d-a877-94503a62c6f8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797065} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de8077c8 5426a961-a6aa-4f7d-a877-94503a62c6f8 /tidb/cdc/default/default/upstream/7364997265750694396 {"id":7364997265750694396,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: d3f75e4e-930d-4baa-a89a-5a75112d2fd7 Info: {"upstream_id":7364997265750694396,"namespace":"default","id":"d3f75e4e-930d-4baa-a89a-5a75112d2fd7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:31:08.338209779+08:00","start_ts":449523761814437889,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523761814437889,"checkpoint_ts":449523761814437889,"checkpoint_time":"2024-05-04 12:31:05.027"} [Sat May 4 12:31:08 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.26012603.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:31:10 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68e93933-f222-4b2b-868f-286a85fbf23b {"id":"68e93933-f222-4b2b-868f-286a85fbf23b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797067} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de80bacc 68e93933-f222-4b2b-868f-286a85fbf23b /tidb/cdc/default/default/upstream/7364997263096637593 {"id":7364997263096637593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68e93933-f222-4b2b-868f-286a85fbf23b {"id":"68e93933-f222-4b2b-868f-286a85fbf23b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797067} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de80bacc 68e93933-f222-4b2b-868f-286a85fbf23b /tidb/cdc/default/default/upstream/7364997263096637593 {"id":7364997263096637593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68e93933-f222-4b2b-868f-286a85fbf23b {"id":"68e93933-f222-4b2b-868f-286a85fbf23b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797067} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de80bacc 68e93933-f222-4b2b-868f-286a85fbf23b /tidb/cdc/default/default/upstream/7364997263096637593 {"id":7364997263096637593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:31:11 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f497b9c-e708-4414-8ff9-cb78a6d50a84 {"id":"0f497b9c-e708-4414-8ff9-cb78a6d50a84","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797068} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de7e84cb 0f497b9c-e708-4414-8ff9-cb78a6d50a84 /tidb/cdc/default/default/upstream/7364997267350383072 {"id":7364997267350383072,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f497b9c-e708-4414-8ff9-cb78a6d50a84 {"id":"0f497b9c-e708-4414-8ff9-cb78a6d50a84","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797068} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de7e84cb 0f497b9c-e708-4414-8ff9-cb78a6d50a84 /tidb/cdc/default/default/upstream/7364997267350383072 {"id":7364997267350383072,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f497b9c-e708-4414-8ff9-cb78a6d50a84 {"id":"0f497b9c-e708-4414-8ff9-cb78a6d50a84","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797068} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de7e84cb 0f497b9c-e708-4414-8ff9-cb78a6d50a84 /tidb/cdc/default/default/upstream/7364997267350383072 {"id":7364997267350383072,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 12:31:11 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.26362638.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8400; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ***************** properties ***************** "mysql.host"="127.0.0.1" "scanproportion"="0" "fieldcount"="3" "mysql.db"="consistent_replicate_ddl" "requestdistribution"="uniform" "readproportion"="0" "operationcount"="0" "insertproportion"="0" "mysql.port"="4000" "workload"="core" "readallfields"="true" "updateproportion"="0" "mysql.user"="root" "recordcount"="50" "threadcount"="10" "dotransactions"="false" ********************************************** Run finished, takes 11.618256ms INSERT - Takes(s): 0.0, Count: 50, OPS: 8049.0, Avg(us): 2105, Min(us): 935, Max(us): 5964, 95th(us): 6000, 99th(us): 6000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:31:11 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca3501e3-eb93-4be2-831d-46349739f927 {"id":"ca3501e3-eb93-4be2-831d-46349739f927","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797069} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de8394cf ca3501e3-eb93-4be2-831d-46349739f927 /tidb/cdc/default/default/upstream/7364997261967339875 {"id":7364997261967339875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca3501e3-eb93-4be2-831d-46349739f927 {"id":"ca3501e3-eb93-4be2-831d-46349739f927","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797069} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de8394cf ca3501e3-eb93-4be2-831d-46349739f927 /tidb/cdc/default/default/upstream/7364997261967339875 {"id":7364997261967339875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca3501e3-eb93-4be2-831d-46349739f927 {"id":"ca3501e3-eb93-4be2-831d-46349739f927","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797069} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de8394cf ca3501e3-eb93-4be2-831d-46349739f927 /tidb/cdc/default/default/upstream/7364997261967339875 {"id":7364997261967339875,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8400 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:31:14 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8106f167-0f14-4ed1-bf9f-75e26f894c34 {"id":"8106f167-0f14-4ed1-bf9f-75e26f894c34","address":"127.0.0.1:8400","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797071} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f41de7eae8e 8106f167-0f14-4ed1-bf9f-75e26f894c34 /tidb/cdc/default/default/upstream/7364997259209530865 {"id":7364997259209530865,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8106f167-0f14-4ed1-bf9f-75e26f894c34 {"id":"8106f167-0f14-4ed1-bf9f-75e26f894c34","address":"127.0.0.1:8400","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797071} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f41de7eae8e 8106f167-0f14-4ed1-bf9f-75e26f894c34 /tidb/cdc/default/default/upstream/7364997259209530865 {"id":7364997259209530865,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8106f167-0f14-4ed1-bf9f-75e26f894c34 {"id":"8106f167-0f14-4ed1-bf9f-75e26f894c34","address":"127.0.0.1:8400","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797071} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f41de7eae8e 8106f167-0f14-4ed1-bf9f-75e26f894c34 /tidb/cdc/default/default/upstream/7364997259209530865 {"id":7364997259209530865,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2698.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306 -c test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/conf/up.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7364997267350383072,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306","create_time":"2024-05-04T12:31:14.595470437+08:00","start_ts":449523764292222981,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523764292222981,"checkpoint_ts":449523764292222981,"checkpoint_time":"2024-05-04 12:31:14.479"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077b5a200016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:1183, start at 2024-05-04 12:31:14.091603246 +0800 CST m=+5.097341770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:14.099 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:14.056 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:14.056 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077b5a200016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:1183, start at 2024-05-04 12:31:14.091603246 +0800 CST m=+5.097341770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:14.099 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:14.056 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:14.056 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077b5d340004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:1266, start at 2024-05-04 12:31:14.25641187 +0800 CST m=+5.187433630 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:14.264 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:14.253 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:14.253 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2729.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:4000 -c test-2 --server http://127.0.0.1:8400 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bdr_mode/conf/down.toml table consistent_replicate_ddl.usertable1 not exists for 2-th check, retry later table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later Create changefeed successfully! ID: test-2 Info: {"upstream_id":7364997259209530865,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:4000","create_time":"2024-05-04T12:31:16.498203931+08:00","start_ts":449523764788985863,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523764788985863,"checkpoint_ts":449523764788985863,"checkpoint_time":"2024-05-04 12:31:16.374"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2633.out cli tso query --pd=http://127.0.0.1:2379 table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later table consistent_replicate_ddl.usertable1 exists table consistent_replicate_ddl.usertable2 not exists for 1-th check, retry later + set +x start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x + tso='449523765125054465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523765125054465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "scanproportion"="0" "mysql.port"="4000" "readproportion"="0" "mysql.db"="sink_retry" "recordcount"="10" "mysql.user"="root" "updateproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "insertproportion"="0" "readallfields"="true" "operationcount"="0" "workload"="core" "mysql.host"="127.0.0.1" "threadcount"="2" ********************************************** Run finished, takes 7.718162ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2356.8, Avg(us): 1472, Min(us): 874, Max(us): 3402, 95th(us): 4000, 99th(us): 4000 [Sat May 4 12:31:19 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.26872689.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 2-th check, retry later table consistent_replicate_ddl.usertable2 not exists for 2-th check, retry later Verifying downstream PD is started... table test.finish_mark not exists for 3-th check, retry later table consistent_replicate_ddl.usertable2 exists table consistent_replicate_ddl.usertable3 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:31:22 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a254dc4e-6c5a-496e-a293-9f015d963203 {"id":"a254dc4e-6c5a-496e-a293-9f015d963203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41deb2c4e8 a254dc4e-6c5a-496e-a293-9f015d963203 /tidb/cdc/default/default/upstream/7364997317036686571 {"id":7364997317036686571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a254dc4e-6c5a-496e-a293-9f015d963203 {"id":"a254dc4e-6c5a-496e-a293-9f015d963203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41deb2c4e8 a254dc4e-6c5a-496e-a293-9f015d963203 /tidb/cdc/default/default/upstream/7364997317036686571 {"id":7364997317036686571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a254dc4e-6c5a-496e-a293-9f015d963203 {"id":"a254dc4e-6c5a-496e-a293-9f015d963203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41deb2c4e8 a254dc4e-6c5a-496e-a293-9f015d963203 /tidb/cdc/default/default/upstream/7364997317036686571 {"id":7364997317036686571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2744.out cli changefeed create --start-ts=449523765125054465 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: 12e2ea1e-66c9-45d5-9f6e-fbd0d0c1be5f Info: {"upstream_id":7364997317036686571,"namespace":"default","id":"12e2ea1e-66c9-45d5-9f6e-fbd0d0c1be5f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T12:31:22.876557344+08:00","start_ts":449523765125054465,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523765125054465,"checkpoint_ts":449523765125054465,"checkpoint_time":"2024-05-04 12:31:17.656"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table consistent_replicate_ddl.usertable3 not exists for 2-th check, retry later table test.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table test.finish_mark not exists for 5-th check, retry later table consistent_replicate_ddl.usertable3 exists table consistent_replicate_ddl.usertable_bak not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark exists check table exists success check diff successfully table consistent_replicate_ddl.usertable_bak not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... table consistent_replicate_ddl.usertable_bak exists cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:31:29 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077c4dcc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:6364, start at 2024-05-04 12:31:29.676314999 +0800 CST m=+5.395513640 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:29.683 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:29.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:29.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077c4dcc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:6364, start at 2024-05-04 12:31:29.676314999 +0800 CST m=+5.395513640 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:29.683 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:29.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:29.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077c4f440003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:6426, start at 2024-05-04 12:31:29.746635819 +0800 CST m=+5.397457443 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:29.753 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:29.745 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:29.745 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sat May 4 12:31:34 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.76747676.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:31:36 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.28412843.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server2 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:31:37 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3d9b9f0-2fe8-4e09-9493-35032171ecf6 {"id":"b3d9b9f0-2fe8-4e09-9493-35032171ecf6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797094} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dee681d3 b3d9b9f0-2fe8-4e09-9493-35032171ecf6 /tidb/cdc/default/default/upstream/7364997369934932005 {"id":7364997369934932005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3d9b9f0-2fe8-4e09-9493-35032171ecf6 {"id":"b3d9b9f0-2fe8-4e09-9493-35032171ecf6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797094} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dee681d3 b3d9b9f0-2fe8-4e09-9493-35032171ecf6 /tidb/cdc/default/default/upstream/7364997369934932005 {"id":7364997369934932005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3d9b9f0-2fe8-4e09-9493-35032171ecf6 {"id":"b3d9b9f0-2fe8-4e09-9493-35032171ecf6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797094} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dee681d3 b3d9b9f0-2fe8-4e09-9493-35032171ecf6 /tidb/cdc/default/default/upstream/7364997369934932005 {"id":7364997369934932005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table processor_stop_delay.t not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:31:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/93e29eae-e4e5-4640-be04-13745a4a60a9 {UpstreamID:7364997263096637593 Namespace:default ID:93e29eae-e4e5-4640-be04-13745a4a60a9 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:31:10.303414952 +0800 CST StartTs:449523763169198082 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033d9950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523763195412481} {CheckpointTs:449523770653409284 MinTableBarrierTs:449523770653409284 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449523770653409284, checkpointTs: 449523770653409284, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/593979cb-1411-4e98-acbc-ffda2122ebff {"id":"593979cb-1411-4e98-acbc-ffda2122ebff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797097} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de80bc1c 593979cb-1411-4e98-acbc-ffda2122ebff /tidb/cdc/default/default/changefeed/info/93e29eae-e4e5-4640-be04-13745a4a60a9 {"upstream-id":7364997263096637593,"namespace":"default","changefeed-id":"93e29eae-e4e5-4640-be04-13745a4a60a9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:31:10.303414952+08:00","start-ts":449523763169198082,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523763195412481} /tidb/cdc/default/default/changefeed/status/93e29eae-e4e5-4640-be04-13745a4a60a9 {"checkpoint-ts":449523770653409284,"min-table-barrier-ts":449523770653409284,"admin-job-type":0} /tidb/cdc/default/default/task/position/593979cb-1411-4e98-acbc-ffda2122ebff/93e29eae-e4e5-4640-be04-13745a4a60a9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997263096637593 {"id":7364997263096637593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/93e29eae-e4e5-4640-be04-13745a4a60a9 {UpstreamID:7364997263096637593 Namespace:default ID:93e29eae-e4e5-4640-be04-13745a4a60a9 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:31:10.303414952 +0800 CST StartTs:449523763169198082 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033d9950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523763195412481} {CheckpointTs:449523770653409284 MinTableBarrierTs:449523770653409284 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449523770653409284, checkpointTs: 449523770653409284, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/593979cb-1411-4e98-acbc-ffda2122ebff {"id":"593979cb-1411-4e98-acbc-ffda2122ebff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797097} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de80bc1c 593979cb-1411-4e98-acbc-ffda2122ebff /tidb/cdc/default/default/changefeed/info/93e29eae-e4e5-4640-be04-13745a4a60a9 {"upstream-id":7364997263096637593,"namespace":"default","changefeed-id":"93e29eae-e4e5-4640-be04-13745a4a60a9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:31:10.303414952+08:00","start-ts":449523763169198082,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523763195412481} /tidb/cdc/default/default/changefeed/status/93e29eae-e4e5-4640-be04-13745a4a60a9 {"checkpoint-ts":449523770653409284,"min-table-barrier-ts":449523770653409284,"admin-job-type":0} /tidb/cdc/default/default/task/position/593979cb-1411-4e98-acbc-ffda2122ebff/93e29eae-e4e5-4640-be04-13745a4a60a9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997263096637593 {"id":7364997263096637593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/93e29eae-e4e5-4640-be04-13745a4a60a9 {UpstreamID:7364997263096637593 Namespace:default ID:93e29eae-e4e5-4640-be04-13745a4a60a9 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:31:10.303414952 +0800 CST StartTs:449523763169198082 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033d9950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523763195412481} {CheckpointTs:449523770653409284 MinTableBarrierTs:449523770653409284 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449523770653409284, checkpointTs: 449523770653409284, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449523770653409285, checkpointTs: 449523770653409284, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/593979cb-1411-4e98-acbc-ffda2122ebff {"id":"593979cb-1411-4e98-acbc-ffda2122ebff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797097} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41de80bc1c 593979cb-1411-4e98-acbc-ffda2122ebff /tidb/cdc/default/default/changefeed/info/93e29eae-e4e5-4640-be04-13745a4a60a9 {"upstream-id":7364997263096637593,"namespace":"default","changefeed-id":"93e29eae-e4e5-4640-be04-13745a4a60a9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:31:10.303414952+08:00","start-ts":449523763169198082,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stu+ grep -q 'etcd info' ck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523763195412481} /tidb/cdc/default/default/changefeed/status/93e29eae-e4e5-4640-be04-13745a4a60a9 {"checkpoint-ts":449523770653409284,"min-table-barrier-ts":449523770653409284,"admin-job-type":0} /tidb/cdc/default/default/task/position/593979cb-1411-4e98-acbc-ffda2122ebff/93e29eae-e4e5-4640-be04-13745a4a60a9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997263096637593 {"id":7364997263096637593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table processor_stop_delay.t exists check diff failed 1-th time, retry later check diff successfully start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff successfully Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 33.89 secs (110057660 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:31:45 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... check diff failed 3-th time, retry later Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: mysql... <<================= wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:31:56 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 9.14 secs (408124636 bytes/sec) [Pipeline] { table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "scanproportion"="0" "recordcount"="10" "mysql.user"="root" "mysql.port"="4000" "workload"="core" "threadcount"="2" "dotransactions"="false" "readallfields"="true" "updateproportion"="0" "insertproportion"="0" "operationcount"="0" "mysql.db"="sink_retry" "readproportion"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" ********************************************** Run finished, takes 10.620965ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3995.4, Avg(us): 2052, Min(us): 438, Max(us): 8074, 95th(us): 9000, 99th(us): 9000 table tidb_cdc.syncpoint_v1 exists table tidb_cdc.syncpoint_v1 does not exists table bdr_mode.finish_mark exists check diff failed 1-th time, retry later [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077dc9c80003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq, pid:3796, start at 2024-05-04 12:31:53.971738241 +0800 CST m=+5.309548112 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:53.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:53.970 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:53.970 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077dc9c80003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq, pid:3796, start at 2024-05-04 12:31:53.971738241 +0800 CST m=+5.309548112 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:53.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:53.970 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:53.970 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077dca7c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq, pid:3885, start at 2024-05-04 12:31:54.036680782 +0800 CST m=+5.325627435 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:33:54.043 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:31:54.015 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:21:54.015 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later The 1 times to try to start tidb cluster... [Sat May 4 12:31:58 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.52575259.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:32:01 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:32:02 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53628ae9-0b33-43ff-8a60-7f30d1cb96e9 {"id":"53628ae9-0b33-43ff-8a60-7f30d1cb96e9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797119} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df45e4cd 53628ae9-0b33-43ff-8a60-7f30d1cb96e9 /tidb/cdc/default/default/upstream/7364997485419021485 {"id":7364997485419021485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53628ae9-0b33-43ff-8a60-7f30d1cb96e9 {"id":"53628ae9-0b33-43ff-8a60-7f30d1cb96e9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797119} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df45e4cd 53628ae9-0b33-43ff-8a60-7f30d1cb96e9 /tidb/cdc/default/default/upstream/7364997485419021485 {"id":7364997485419021485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53628ae9-0b33-43ff-8a60-7f30d1cb96e9 {"id":"53628ae9-0b33-43ff-8a60-7f30d1cb96e9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797119} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df45e4cd 53628ae9-0b33-43ff-8a60-7f30d1cb96e9 /tidb/cdc/default/default/upstream/7364997485419021485 {"id":7364997485419021485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5320.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-manager Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7364997485419021485,"namespace":"default","id":"ddl-manager","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-04T12:32:02.482384569+08:00","start_ts":449523776846299138,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523776846299138,"checkpoint_ts":449523776846299138,"checkpoint_time":"2024-05-04 12:32:02.369"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077efe180005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87, pid:4133, start at 2024-05-04 12:32:13.71231025 +0800 CST m=+5.368491717 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:13.719 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:13.702 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:13.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077efe180005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87, pid:4133, start at 2024-05-04 12:32:13.71231025 +0800 CST m=+5.368491717 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:13.719 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:13.702 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:13.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077efeb40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0df8h-dld9d-g1n87, pid:4225, start at 2024-05-04 12:32:13.757538405 +0800 CST m=+5.325080890 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:13.765 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:13.741 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:13.741 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 12:32:18 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.54645466.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077f6bf40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:9027, start at 2024-05-04 12:32:20.757981195 +0800 CST m=+5.438353859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:20.766 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:20.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:20.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:32:21 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8697fd79-24c4-4573-9e09-d752761c9dc5 {"id":"8697fd79-24c4-4573-9e09-d752761c9dc5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797139} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df92c6ce 8697fd79-24c4-4573-9e09-d752761c9dc5 /tidb/cdc/default/default/upstream/7364997575127988612 {"id":7364997575127988612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8697fd79-24c4-4573-9e09-d752761c9dc5 {"id":"8697fd79-24c4-4573-9e09-d752761c9dc5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797139} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df92c6ce 8697fd79-24c4-4573-9e09-d752761c9dc5 /tidb/cdc/default/default/upstream/7364997575127988612 {"id":7364997575127988612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8697fd79-24c4-4573-9e09-d752761c9dc5 {"id":"8697fd79-24c4-4573-9e09-d752761c9dc5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797139} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df92c6ce 8697fd79-24c4-4573-9e09-d752761c9dc5 /tidb/cdc/default/default/upstream/7364997575127988612 {"id":7364997575127988612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5519.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /tmp/tidb_cdc_test/cdc/pulsar_test.toml wait process 5262 exit for 1-th time... wait process 5262 exit for 2-th time... wait process 5262 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5262) - No such process wait process 5262 exit for 4-th time... process 5262 already exit [Sat May 4 12:32:20 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.54135415.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: 5b7a12ac-27cf-437e-ac62-b568b38d950d Info: {"upstream_id":7364997575127988612,"namespace":"default","id":"5b7a12ac-27cf-437e-ac62-b568b38d950d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:32:22.209405515+08:00","start_ts":449523782019448836,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523782019448836,"checkpoint_ts":449523782019448836,"checkpoint_time":"2024-05-04 12:32:22.103"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 23.77 secs (156931067 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077f6bf40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:9027, start at 2024-05-04 12:32:20.757981195 +0800 CST m=+5.438353859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:20.766 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:20.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:20.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077f6e380013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-cfl7x-khkls-f2crh, pid:9107, start at 2024-05-04 12:32:20.904229756 +0800 CST m=+5.533057228 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:20.914 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:20.878 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:20.878 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... + set +x go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/text v0.14.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:32:24 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364997485419021485 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:32:02.482384569 +0800 CST StartTs:449523776846299138 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004220cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523776872513538} {CheckpointTs:449523777528135719 MinTableBarrierTs:449523777528135719 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0005383f-1e0f-4758-8a7d-eafd569207b5 {"id":"0005383f-1e0f-4758-8a7d-eafd569207b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df45e7b3 0005383f-1e0f-4758-8a7d-eafd569207b5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364997485419021485,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:32:02.482384569+08:00","start-ts":449523776846299138,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523776872513538} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449523777528135719,"min-table-barrier-ts":449523777528135719,"admin-job-type":0} /tidb/cdc/default/default/task/position/0005383f-1e0f-4758-8a7d-eafd569207b5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997485419021485 {"id":7364997485419021485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364997485419021485 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:32:02.482384569 +0800 CST StartTs:449523776846299138 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004220cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523776872513538} {CheckpointTs:449523777528135719 MinTableBarrierTs:449523777528135719 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0005383f-1e0f-4758-8a7d-eafd569207b5 {"id":"0005383f-1e0f-4758-8a7d-eafd569207b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df45e7b3 0005383f-1e0f-4758-8a7d-eafd569207b5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364997485419021485,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:32:02.482384569+08:00","start-ts":449523776846299138,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523776872513538} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449523777528135719,"min-table-barrier-ts":449523777528135719,"admin-job-type":0} /tidb/cdc/default/default/task/position/0005383f-1e0f-4758-8a7d-eafd569207b5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997485419021485 + grep -q 'failed to get info:' {"id":7364997485419021485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364997485419021485 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:32:02.482384569 +0800 CST StartTs:449523776846299138 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004220cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523776872513538} {CheckpointTs:449523777528135719 MinTableBarrierTs:449523777528135719 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0005383f-1e0f-4758-8a7d-eafd569207b5 {"id":"0005383f-1e0f-4758-8a7d-eafd569207b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df45e7b3 0005383f-1e0f-4758-8a7d-eafd569207b5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364997485419021485,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:32:02.482384569+08:00","start-ts":449523776846299138,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523776872513538} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449523777528135719,"min-table-barrier-ts":449523777528135719,"admin-job-type":0} /tidb/cdc/default/default/task/position/0005383f-1e0f-4758-8a7d-eafd569207b5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997485419021485 + grep -q 'etcd info' {"id":7364997485419021485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Sat May 4 12:32:24 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.54765478.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10480.out cli tso query --pd=http://127.0.0.1:2379 < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:32:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364997485419021485 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:32:02.482384569 +0800 CST StartTs:449523776846299138 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004220cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523776872513538} {CheckpointTs:449523777528135719 MinTableBarrierTs:449523777528135719 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0005383f-1e0f-4758-8a7d-eafd569207b5 {"id":"0005383f-1e0f-4758-8a7d-eafd569207b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df45e7b3 0005383f-1e0f-4758-8a7d-eafd569207b5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364997485419021485,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:32:02.482384569+08:00","start-ts":449523776846299138,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523776872513538} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449523777672052744,"min-table-barrier-ts":449523777672052744,"admin-job-type":0} /tidb/cdc/default/default/task/position/0005383f-1e0f-4758-8a7d-eafd569207b5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997485419021485 {"id":7364997485419021485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364997485419021485 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:32:02.482384569 +0800 CST StartTs:449523776846299138 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004220cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523776872513538} {CheckpointTs:449523777528135719 MinTableBarrierTs:449523777528135719 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0005383f-1e0f-4758-8a7d-eafd569207b5 {"id":"0005383f-1e0f-4758-8a7d-eafd569207b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df45e7b3 0005383f-1e0f-4758-8a7d-eafd569207b5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364997485419021485,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:32:02.482384569+08:00","start-ts":449523776846299138,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523776872513538} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449523777672052744,"min-table-barrier-ts":449523777672052744,"admin-job-type":0} /tidb/cdc/default/default/task/position/0005383f-1e0f-4758-8a7d-eafd569207b5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997485419021485 + grep -q 'failed to get info:' {"id":7364997485419021485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364997485419021485 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:32:02.482384569 +0800 CST StartTs:449523776846299138 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004220cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523776872513538} {CheckpointTs:449523777528135719 MinTableBarrierTs:449523777528135719 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0005383f-1e0f-4758-8a7d-eafd569207b5 {"id":"0005383f-1e0f-4758-8a7d-eafd569207b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41df45e7b3 0005383f-1e0f-4758-8a7d-eafd569207b5 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364997485419021485,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:32:02.482384569+08:00","start-ts":449523776846299138,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523776872513538} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449523777672052744,"min-table-barrier-ts":449523777672052744,"admin-job-type":0} /tidb/cdc/default/default/task/position/0005383f-1e0f-4758-8a7d-eafd569207b5/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997485419021485 {"id":7364997485419021485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/google/uuid v1.6.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/google/btree v1.1.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading golang.org/x/time v0.5.0 go: downloading golang.org/x/tools v0.20.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading google.golang.org/api v0.170.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/dolthub/maphash v0.1.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 + set +x + tso='449523783036567553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523783036567553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:32:27 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1051810520.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 2-th check, retry later table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:32:26 CST 2024] <<<<<< run test case sink_retry success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077fbf1c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx, pid:3964, start at 2024-05-04 12:32:26.091483177 +0800 CST m=+5.350155094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:26.099 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:26.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:26.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077fbf1c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx, pid:3964, start at 2024-05-04 12:32:26.091483177 +0800 CST m=+5.350155094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:26.099 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:26.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:26.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d077fc1540014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx, pid:4017, start at 2024-05-04 12:32:26.216623191 +0800 CST m=+5.414574808 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:26.226 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:26.197 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:26.197 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:32:30 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3cacb4e-ad13-4ffd-b000-d5c9cca5f39e {"id":"c3cacb4e-ad13-4ffd-b000-d5c9cca5f39e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797148} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dfae0bd4 c3cacb4e-ad13-4ffd-b000-d5c9cca5f39e /tidb/cdc/default/default/upstream/7364997603834821093 {"id":7364997603834821093,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3cacb4e-ad13-4ffd-b000-d5c9cca5f39e {"id":"c3cacb4e-ad13-4ffd-b000-d5c9cca5f39e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797148} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dfae0bd4 c3cacb4e-ad13-4ffd-b000-d5c9cca5f39e /tidb/cdc/default/default/upstream/7364997603834821093 {"id":7364997603834821093,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3cacb4e-ad13-4ffd-b000-d5c9cca5f39e {"id":"c3cacb4e-ad13-4ffd-b000-d5c9cca5f39e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797148} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dfae0bd4 c3cacb4e-ad13-4ffd-b000-d5c9cca5f39e /tidb/cdc/default/default/upstream/7364997603834821093 {"id":7364997603834821093,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10576.out cli changefeed create --start-ts=449523783036567553 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 [Sat May 4 12:32:31 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.52715273.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: c683fb77-b1a1-4ba5-ba61-090292cc6c09 Info: {"upstream_id":7364997603834821093,"namespace":"default","id":"c683fb77-b1a1-4ba5-ba61-090292cc6c09","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:32:31.158540672+08:00","start_ts":449523783036567553,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523783036567553,"checkpoint_ts":449523783036567553,"checkpoint_time":"2024-05-04 12:32:25.983"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd table ddl_manager.finish_mark not exists for 4-th check, retry later + set +x table ddl_manager.finish_mark not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:32:34 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aabffaed-6633-4aa5-92be-3298b5216f8e {"id":"aabffaed-6633-4aa5-92be-3298b5216f8e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dfc307d2 aabffaed-6633-4aa5-92be-3298b5216f8e /tidb/cdc/default/default/upstream/7364997616107786367 {"id":7364997616107786367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aabffaed-6633-4aa5-92be-3298b5216f8e {"id":"aabffaed-6633-4aa5-92be-3298b5216f8e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dfc307d2 aabffaed-6633-4aa5-92be-3298b5216f8e /tidb/cdc/default/default/upstream/7364997616107786367 {"id":7364997616107786367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aabffaed-6633-4aa5-92be-3298b5216f8e {"id":"aabffaed-6633-4aa5-92be-3298b5216f8e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dfc307d2 aabffaed-6633-4aa5-92be-3298b5216f8e /tidb/cdc/default/default/upstream/7364997616107786367 {"id":7364997616107786367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 12:32:34 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.53285330.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 6-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:32:37 GMT < Content-Length: 1259 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43f5c3e3-1e32-4807-b068-667397371abc {"id":"43f5c3e3-1e32-4807-b068-667397371abc","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797154} /tidb/cdc/default/__cdc_meta__/capture/aabffaed-6633-4aa5-92be-3298b5216f8e {"id":"aabffaed-6633-4aa5-92be-3298b5216f8e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dfc307d2 aabffaed-6633-4aa5-92be-3298b5216f8e /tidb/cdc/default/__cdc_meta__/owner/22318f41dfc30805 43f5c3e3-1e32-4807-b068-667397371abc /tidb/cdc/default/default/upstream/7364997616107786367 {"id":7364997616107786367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43f5c3e3-1e32-4807-b068-667397371abc {"id":"43f5c3e3-1e32-4807-b068-667397371abc","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797154} /tidb/cdc/default/__cdc_meta__/capture/aabffaed-6633-4aa5-92be-3298b5216f8e {"id":"aabffaed-6633-4aa5-92be-3298b5216f8e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dfc307d2 aabffaed-6633-4aa5-92be-3298b5216f8e /tidb/cdc/default/__cdc_meta__/owner/22318f41dfc30805 43f5c3e3-1e32-4807-b068-667397371abc /tidb/cdc/default/default/upstream/7364997616107786367 {"id":7364997616107786367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43f5c3e3-1e32-4807-b068-667397371abc {"id":"43f5c3e3-1e32-4807-b068-667397371abc","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797154} /tidb/cdc/default/__cdc_meta__/capture/aabffaed-6633-4aa5-92be-3298b5216f8e {"id":"aabffaed-6633-4aa5-92be-3298b5216f8e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41dfc307d2 aabffaed-6633-4aa5-92be-3298b5216f8e /tidb/cdc/default/__cdc_meta__/owner/22318f41dfc30805 43f5c3e3-1e32-4807-b068-667397371abc /tidb/cdc/default/default/upstream/7364997616107786367 {"id":7364997616107786367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 7-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table capture_suicide_while_balance_table.t1 exists table capture_suicide_while_balance_table.t2 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 8-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 15.53 secs (240228459 bytes/sec) [Pipeline] { [Pipeline] cache Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 9-th check, retry later table capture_suicide_while_balance_table.t2 exists table capture_suicide_while_balance_table.t3 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 10-th check, retry later table capture_suicide_while_balance_table.t3 exists table capture_suicide_while_balance_table.t4 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 11-th check, retry later table capture_suicide_while_balance_table.t4 exists table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later table partition_table.t2 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 12-th check, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 140 100 35 100 105 686 2060 --:--:-- --:--:-- --:--:-- 2100 { "status": true, "message": "" table partition_table.t2 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 13-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 7.22 secs (516644892 bytes/sec) [Pipeline] { [Pipeline] stage [Pipeline] { (Test) }lease 22318f41dfc30805 revoked [Pipeline] stage [Pipeline] { (Test) [Pipeline] sh check diff failed 1-th time, retry later [Pipeline] sh table partition_table.t2 not exists for 4-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=349495d8-c782-4820-b838-739ddd19b1de BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-0q39v GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-0q39v pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=37b193ba-6d96-40ed-88a0-0235b6496617 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-d59k5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-d59k5 pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] sh table ddl_manager.finish_mark not exists for 14-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=4252a8d8-796d-4883-8dc6-7ecb5245496c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-wmgf0 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-wmgf0 pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=71b9ea62-c08a-4c6d-b4e5-af51b6397424 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-gjgv5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w pingcap_tiflow_pull_cdc_integration_test_1731-gjgv5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: mysql... <<================= [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G12 [Pipeline] { [Pipeline] // withCredentials [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=b9834184-37a1-429e-8724-e31f8edb302e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-xjtcb GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh pingcap_tiflow_pull_cdc_integration_test_1731-xjtcb GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: mysql... <<================= [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0781421c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:3796, start at 2024-05-04 12:32:50.866908282 +0800 CST m=+5.314106677 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:50.874 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:50.873 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:50.873 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0781421c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:3796, start at 2024-05-04 12:32:50.866908282 +0800 CST m=+5.314106677 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:50.874 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:50.873 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:50.873 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078142140015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:3872, start at 2024-05-04 12:32:50.856738242 +0800 CST m=+5.256910634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:34:50.863 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:32:50.821 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:22:50.821 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 15-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... table partition_table.finish_mark not exists for 2-th check, retry later Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... The 1 times to try to start tidb cluster... The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 16-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5228.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully table partition_table.finish_mark not exists for 3-th check, retry later + set +x + tso='449523790963802113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523790963802113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 17-th check, retry later wait process cdc.test exit for 2-th time... The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ***************** properties ***************** "readallfields"="true" "mysql.user"="root" "updateproportion"="0" "recordcount"="20" "dotransactions"="false" "requestdistribution"="uniform" "workload"="core" "operationcount"="0" "mysql.db"="changefeed_error" "insertproportion"="0" "readproportion"="0" "threadcount"="4" "mysql.host"="127.0.0.1" "mysql.port"="4000" "scanproportion"="0" ********************************************** Run finished, takes 9.700429ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3691.3, Avg(us): 1841, Min(us): 1076, Max(us): 4286, 95th(us): 5000, 99th(us): 5000 [Sat May 4 12:32:58 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.52855287.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:32:59 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> table partition_table.finish_mark not exists for 4-th check, retry later Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // timeout [Pipeline] } [Pipeline] { table ddl_manager.finish_mark not exists for 18-th check, retry later [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { table partition_table.finish_mark not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:01 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 {"id":"4302e5ba-3efd-42e0-9810-67b9842827e0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797179} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023fcf9 4302e5ba-3efd-42e0-9810-67b9842827e0 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 {"id":"4302e5ba-3efd-42e0-9810-67b9842827e0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797179} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023fcf9 4302e5ba-3efd-42e0-9810-67b9842827e0 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 {"id":"4302e5ba-3efd-42e0-9810-67b9842827e0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797179} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023fcf9 4302e5ba-3efd-42e0-9810-67b9842827e0 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5355.out cli changefeed create --start-ts=449523790963802113 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7364997725240484907,"namespace":"default","id":"changefeed-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T12:33:02.324814344+08:00","start_ts":449523790963802113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523790963802113,"checkpoint_ts":449523790963802113,"checkpoint_time":"2024-05-04 12:32:56.223"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 19-th check, retry later table partition_table.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + info='{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449523790963802113, "checkpoint_time": "2024-05-04 12:32:56.223", "error": { "time": "2024-05-04T12:33:02.435231174+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449523790963801813 is earlier than or equal to GC safepoint at 449523790963802113" } }' + echo '{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449523790963802113, "checkpoint_time": "2024-05-04 12:32:56.223", "error": { "time": "2024-05-04T12:33:02.435231174+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449523790963801813 is earlier than or equal to GC safepoint at 449523790963802113" } }' { "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449523790963802113, "checkpoint_time": "2024-05-04 12:32:56.223", "error": { "time": "2024-05-04T12:33:02.435231174+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449523790963801813 is earlier than or equal to GC safepoint at 449523790963802113" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449523790963802113, '"checkpoint_time":' '"2024-05-04' '12:32:56.223",' '"error":' '{' '"time":' '"2024-05-04T12:33:02.435231174+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449523790963801813 is earlier than or equal to GC safepoint at '449523790963802113"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449523790963802113, '"checkpoint_time":' '"2024-05-04' '12:32:56.223",' '"error":' '{' '"time":' '"2024-05-04T12:33:02.435231174+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449523790963801813 is earlier than or equal to GC safepoint at '449523790963802113"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449523790963801813 is earlier than or equal to GC safepoint at 449523790963802113' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449523790963801813 is earlier than or equal to GC safepoint at 449523790963802113 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5433.out cli changefeed resume -c changefeed-error Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 20-th check, retry later Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table partition_table.finish_mark not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078214f80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:1138, start at 2024-05-04 12:33:04.318854196 +0800 CST m=+5.510657997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:04.328 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:04.318 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:04.318 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 21-th check, retry later + set +x table changefeed_error.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07822a480010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2, pid:1283, start at 2024-05-04 12:33:05.699767946 +0800 CST m=+5.334789890 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:05.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:05.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:05.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078214f80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:1138, start at 2024-05-04 12:33:04.318854196 +0800 CST m=+5.510657997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:04.328 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:04.318 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:04.318 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078214640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:1218, start at 2024-05-04 12:33:04.31026594 +0800 CST m=+5.423847492 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:04.316 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:04.281 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:04.281 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 22-th check, retry later table changefeed_error.usertable exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07822a480010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2, pid:1283, start at 2024-05-04 12:33:05.699767946 +0800 CST m=+5.334789890 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:05.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:05.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:05.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07822a140016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2, pid:1371, start at 2024-05-04 12:33:05.716726638 +0800 CST m=+5.299591990 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:05.723 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:05.718 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:05.718 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 6.10 secs (611960955 bytes/sec) [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] cache [Sat May 4 12:33:10 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.26212623.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 23-th check, retry later check diff successfully ***************** properties ***************** "readallfields"="true" "mysql.user"="root" "mysql.db"="changefeed_error" "insertproportion"="0" "updateproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "workload"="core" "recordcount"="20" "threadcount"="4" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.port"="4000" "scanproportion"="0" "readproportion"="0" ********************************************** Run finished, takes 4.554786ms INSERT - Takes(s): 0.0, Count: 20, OPS: 6156.6, Avg(us): 798, Min(us): 431, Max(us): 1899, 95th(us): 2000, 99th(us): 2000 check diff successfully {"id":"4302e5ba-3efd-42e0-9810-67b9842827e0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797179} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 + [[ /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later [Sat May 4 12:33:11 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.27322734.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 10-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078274880013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w, pid:1297, start at 2024-05-04 12:33:10.46404449 +0800 CST m=+5.385293613 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:10.470 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:10.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:10.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07825c100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb, pid:1250, start at 2024-05-04 12:33:08.90072188 +0800 CST m=+5.242255295 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:08.910 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:08.868 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:08.868 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07825c100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb, pid:1250, start at 2024-05-04 12:33:08.90072188 +0800 CST m=+5.242255295 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:08.910 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:08.868 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:08.868 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07825f24000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-wmgf0-ls35n-ml0rb, pid:1332, start at 2024-05-04 12:33:09.075121399 +0800 CST m=+5.361761528 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:09.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:09.065 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:09.065 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 24-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 + [[ /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table partition_table.finish_mark exists check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078274880013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w, pid:1297, start at 2024-05-04 12:33:10.46404449 +0800 CST m=+5.385293613 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:10.470 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:10.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:10.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07827538000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-gjgv5-1h70n-hzw8w, pid:1385, start at 2024-05-04 12:33:10.493049333 +0800 CST m=+5.319229443 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:10.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:10.478 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:10.478 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078299d80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh, pid:1294, start at 2024-05-04 12:33:12.842131573 +0800 CST m=+5.383573617 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:12.849 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:12.822 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:12.822 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2719.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:13 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33344bd3-e264-4723-afd0-b55b70a08316 {"id":"33344bd3-e264-4723-afd0-b55b70a08316","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797191} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e061eacd 33344bd3-e264-4723-afd0-b55b70a08316 /tidb/cdc/default/default/upstream/7364997798452918721 {"id":7364997798452918721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33344bd3-e264-4723-afd0-b55b70a08316 {"id":"33344bd3-e264-4723-afd0-b55b70a08316","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797191} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e061eacd 33344bd3-e264-4723-afd0-b55b70a08316 /tidb/cdc/default/default/upstream/7364997798452918721 {"id":7364997798452918721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33344bd3-e264-4723-afd0-b55b70a08316 {"id":"33344bd3-e264-4723-afd0-b55b70a08316","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797191} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e061eacd 33344bd3-e264-4723-afd0-b55b70a08316 /tidb/cdc/default/default/upstream/7364997798452918721 {"id":7364997798452918721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x wait process cdc.test exit for 1-th time... 0 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:14 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22487164-ed1d-4843-aea0-751d54056361 {"id":"22487164-ed1d-4843-aea0-751d54056361","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797191} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e061ad66 22487164-ed1d-4843-aea0-751d54056361 /tidb/cdc/default/default/upstream/7364997785982243939 {"id":7364997785982243939,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22487164-ed1d-4843-aea0-751d54056361 {"id":"22487164-ed1d-4843-aea0-751d54056361","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797191} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e061ad66 22487164-ed1d-4843-aea0-751d54056361 /tidb/cdc/default/default/upstream/7364997785982243939 {"id":7364997785982243939,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22487164-ed1d-4843-aea0-751d54056361 {"id":"22487164-ed1d-4843-aea0-751d54056361","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797191} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e061ad66 22487164-ed1d-4843-aea0-751d54056361 /tidb/cdc/default/default/upstream/7364997785982243939 {"id":7364997785982243939,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 25-th check, retry later check_changefeed_state http://127.0.0.1:2379 7363a5e2-51eb-41a2-9d77-9146f2de0c03 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=7363a5e2-51eb-41a2-9d77-9146f2de0c03 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 7363a5e2-51eb-41a2-9d77-9146f2de0c03 -s wait process cdc.test exit for 3-th time... + info='{ "upstream_id": 7364997785982243939, "namespace": "default", "id": "7363a5e2-51eb-41a2-9d77-9146f2de0c03", "state": "normal", "checkpoint_tso": 449523795746619393, "checkpoint_time": "2024-05-04 12:33:14.468", "error": null }' + echo '{ "upstream_id": 7364997785982243939, "namespace": "default", "id": "7363a5e2-51eb-41a2-9d77-9146f2de0c03", "state": "normal", "checkpoint_tso": 449523795746619393, "checkpoint_time": "2024-05-04 12:33:14.468", "error": null }' { "upstream_id": 7364997785982243939, "namespace": "default", "id": "7363a5e2-51eb-41a2-9d77-9146f2de0c03", "state": "normal", "checkpoint_tso": 449523795746619393, "checkpoint_time": "2024-05-04 12:33:14.468", "error": null } ++ echo '{' '"upstream_id":' 7364997785982243939, '"namespace":' '"default",' '"id":' '"7363a5e2-51eb-41a2-9d77-9146f2de0c03",' '"state":' '"normal",' '"checkpoint_tso":' 449523795746619393, '"checkpoint_time":' '"2024-05-04' '12:33:14.468",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364997785982243939, '"namespace":' '"default",' '"id":' '"7363a5e2-51eb-41a2-9d77-9146f2de0c03",' '"state":' '"normal",' '"checkpoint_tso":' 449523795746619393, '"checkpoint_time":' '"2024-05-04' '12:33:14.468",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2703.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:33:15 CST 2024] <<<<<< run test case partition_table success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078299d80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh, pid:1294, start at 2024-05-04 12:33:12.842131573 +0800 CST m=+5.383573617 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:12.849 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:12.822 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:12.822 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07829d040010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh, pid:1379, start at 2024-05-04 12:33:13.041678382 +0800 CST m=+5.500364992 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:13.050 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:13.025 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:13.025 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449523795707297793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523795707297793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:33:15 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.27552757.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 26-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 + [[ /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/4302e5ba-3efd-42e0-9810-67b9842827e0 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later + set +x + tso='449523796078493697 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523796078493697 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:33:17 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.27402742.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2690.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 27-th check, retry later check diff successfully check_safepoint_forward http://127.0.0.1:2379 7364997798452918721 449523796523614208 449523795579895815 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:18 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/daa86605-0a93-4311-a365-20e13574ec5c {"id":"daa86605-0a93-4311-a365-20e13574ec5c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797196} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e070b1d6 daa86605-0a93-4311-a365-20e13574ec5c /tidb/cdc/default/default/upstream/7364997809319206071 {"id":7364997809319206071,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/daa86605-0a93-4311-a365-20e13574ec5c {"id":"daa86605-0a93-4311-a365-20e13574ec5c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797196} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e070b1d6 daa86605-0a93-4311-a365-20e13574ec5c /tidb/cdc/default/default/upstream/7364997809319206071 {"id":7364997809319206071,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/daa86605-0a93-4311-a365-20e13574ec5c {"id":"daa86605-0a93-4311-a365-20e13574ec5c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797196} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e070b1d6 daa86605-0a93-4311-a365-20e13574ec5c /tidb/cdc/default/default/upstream/7364997809319206071 {"id":7364997809319206071,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2810.out cli changefeed create --start-ts=449523795707297793 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ check diff successfully Create changefeed successfully! ID: 0a346e20-7035-4782-8b87-f8752f988b77 Info: {"upstream_id":7364997809319206071,"namespace":"default","id":"0a346e20-7035-4782-8b87-f8752f988b77","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:33:19.436209789+08:00","start_ts":449523795707297793,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523795707297793,"checkpoint_ts":449523795707297793,"checkpoint_time":"2024-05-04 12:33:14.318"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + tso='449523796730707969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523796730707969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:33:19 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.27232725.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' run task successfully check_changefeed_state http://127.0.0.1:2379 42627c8f-363e-4b1e-b2ba-62a9055b9bd5 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=42627c8f-363e-4b1e-b2ba-62a9055b9bd5 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 42627c8f-363e-4b1e-b2ba-62a9055b9bd5 -s + info='{ "upstream_id": 7364997798452918721, "namespace": "default", "id": "42627c8f-363e-4b1e-b2ba-62a9055b9bd5", "state": "stopped", "checkpoint_tso": 449523797047902211, "checkpoint_time": "2024-05-04 12:33:19.432", "error": null }' + echo '{ "upstream_id": 7364997798452918721, "namespace": "default", "id": "42627c8f-363e-4b1e-b2ba-62a9055b9bd5", "state": "stopped", "checkpoint_tso": 449523797047902211, "checkpoint_time": "2024-05-04 12:33:19.432", "error": null }' { "upstream_id": 7364997798452918721, "namespace": "default", "id": "42627c8f-363e-4b1e-b2ba-62a9055b9bd5", "state": "stopped", "checkpoint_tso": 449523797047902211, "checkpoint_time": "2024-05-04 12:33:19.432", "error": null } ++ echo '{' '"upstream_id":' 7364997798452918721, '"namespace":' '"default",' '"id":' '"42627c8f-363e-4b1e-b2ba-62a9055b9bd5",' '"state":' '"stopped",' '"checkpoint_tso":' 449523797047902211, '"checkpoint_time":' '"2024-05-04' '12:33:19.432",' '"error":' null '}' ++ jq -r .state + '[' 0 -eq 50 ']' + sleep 3 + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7364997798452918721, '"namespace":' '"default",' '"id":' '"42627c8f-363e-4b1e-b2ba-62a9055b9bd5",' '"state":' '"stopped",' '"checkpoint_tso":' 449523797047902211, '"checkpoint_time":' '"2024-05-04' '12:33:19.432",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7364997798452918721 wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 28-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:20 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0eac611-260f-4f95-97b9-8b16e359ec5d {"id":"d0eac611-260f-4f95-97b9-8b16e359ec5d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797197} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e07256d5 d0eac611-260f-4f95-97b9-8b16e359ec5d /tidb/cdc/default/default/upstream/7364997812849846074 {"id":7364997812849846074,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0eac611-260f-4f95-97b9-8b16e359ec5d {"id":"d0eac611-260f-4f95-97b9-8b16e359ec5d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797197} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e07256d5 d0eac611-260f-4f95-97b9-8b16e359ec5d /tidb/cdc/default/default/upstream/7364997812849846074 {"id":7364997812849846074,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0eac611-260f-4f95-97b9-8b16e359ec5d {"id":"d0eac611-260f-4f95-97b9-8b16e359ec5d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797197} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e07256d5 d0eac611-260f-4f95-97b9-8b16e359ec5d /tidb/cdc/default/default/upstream/7364997812849846074 {"id":7364997812849846074,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2799.out cli changefeed create --start-ts=449523796078493697 --sink-uri=mysql://normal:123456@127.0.0.1:3306/tidb-txn-mode=pessimistic wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: 225e9ebf-2455-49a6-942b-843291cc542e Info: {"upstream_id":7364997812849846074,"namespace":"default","id":"225e9ebf-2455-49a6-942b-843291cc542e","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/tidb-txn-mode=pessimistic","create_time":"2024-05-04T12:33:20.769288846+08:00","start_ts":449523796078493697,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523796078493697,"checkpoint_ts":449523796078493697,"checkpoint_time":"2024-05-04 12:33:15.734"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) safepoint is unexpected forward: 449523796785758209 449523795579895815 -> 449523797047902210 449523795579895815 run task failed 1-th time, retry later + set +x go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading golang.org/x/time v0.5.0 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:33:21 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/net v0.24.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/text v0.14.0 table ddl_manager.finish_mark not exists for 29-th check, retry later + set +x go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/zap v1.27.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/google/btree v1.1.2 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_safepoint_equal http://127.0.0.1:2379 7364997798452918721 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/sys v0.19.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:22 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38e44ae8-617b-47da-b27c-72e6aea9ac65 {"id":"38e44ae8-617b-47da-b27c-72e6aea9ac65","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797200} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e07b79d7 38e44ae8-617b-47da-b27c-72e6aea9ac65 /tidb/cdc/default/default/upstream/7364997819605925062 {"id":7364997819605925062,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38e44ae8-617b-47da-b27c-72e6aea9ac65 {"id":"38e44ae8-617b-47da-b27c-72e6aea9ac65","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797200} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e07b79d7 38e44ae8-617b-47da-b27c-72e6aea9ac65 /tidb/cdc/default/default/upstream/7364997819605925062 {"id":7364997819605925062,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38e44ae8-617b-47da-b27c-72e6aea9ac65 {"id":"38e44ae8-617b-47da-b27c-72e6aea9ac65","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797200} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e07b79d7 38e44ae8-617b-47da-b27c-72e6aea9ac65 /tidb/cdc/default/default/upstream/7364997819605925062 {"id":7364997819605925062,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2782.out cli changefeed create --start-ts=449523796730707969 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/cloudfoundry/gosigar v1.3.6 check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 12:33:22 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.56845686.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: 62d0c27a-e21c-4e76-b177-d03909d164a4 Info: {"upstream_id":7364997819605925062,"namespace":"default","id":"62d0c27a-e21c-4e76-b177-d03909d164a4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:33:23.300632814+08:00","start_ts":449523796730707969,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523796730707969,"checkpoint_ts":449523796730707969,"checkpoint_time":"2024-05-04 12:33:18.222"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/stretchr/testify v1.9.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/snappy v0.0.4 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/spkg/bom v1.0.0 table ddl_manager.finish_mark not exists for 30-th check, retry later go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078324300013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx, pid:6640, start at 2024-05-04 12:33:21.717969222 +0800 CST m=+5.387288283 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:21.725 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:21.725 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:21.725 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078324300013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx, pid:6640, start at 2024-05-04 12:33:21.717969222 +0800 CST m=+5.387288283 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:21.725 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:21.725 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:21.725 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078324d40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-91kqp-8067w-jq2sx, pid:6711, start at 2024-05-04 12:33:21.752272127 +0800 CST m=+5.353886961 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:21.759 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:21.717 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:21.717 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da + set +x go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading go.uber.org/zap v1.27.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/text v0.14.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/stretchr/testify v1.9.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/golang/snappy v0.0.4 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/klauspost/compress v1.17.8 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/apache/thrift v0.16.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/compute v1.25.1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/31e2ed57-0b26-40e2-aa19-b62105e0404e {"id":"31e2ed57-0b26-40e2-aa19-b62105e0404e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797202} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023fdeb 31e2ed57-0b26-40e2-aa19-b62105e0404e /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7364997725240484907,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T12:33:02.324814344+08:00","start-ts":449523790963802113,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T12:33:23.177302241+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523798028845057} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449523794476531713,"min-table-barrier-ts":449523794476531713,"admin-job-type":1} /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/31e2ed57-0b26-40e2-aa19-b62105e0404e go: downloading github.com/dustin/go-humanize v1.0.1 {"id":"31e2ed57-0b26-40e2-aa19-b62105e0404e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797202} go: downloading github.com/golang/glog v1.2.0 /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 go: downloading github.com/rivo/uniseg v0.4.7 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b /tidb/cdc/default/__cdc_meta__/owner/22318f41e023fdeb 31e2ed57-0b26-40e2-aa19-b62105e0404e go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 /tidb/cdc/default/default/changefeed/info/changefeed-error go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading go.opentelemetry.io/otel v1.24.0 {"upstream-id":7364997725240484907,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T12:33:02.324814344+08:00","start-ts":449523790963802113,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T12:33:23.177302241+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523798028845057} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449523794476531713,"min-table-barrier-ts":449523794476531713,"admin-job-type":1} /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/31e2ed57-0b26-40e2-aa19-b62105e0404e {"id":"31e2ed57-0b26-40e2-aa19-b62105e0404e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797202} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023fdeb 31e2ed57-0b26-40e2-aa19-b62105e0404e /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7364997725240484907,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T12:33:02.324814344+08:00","start-ts":449523790963802113,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T12:33:23.177302241+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523798028845057} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449523794476531713,"min-table-barrier-ts":449523794476531713,"admin-job-type":1} /tidb/cdc/default/default/upstream/7364997725240484907 + grep -q 'etcd info' {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449523794476531713, "checkpoint_time": "2024-05-04 12:33:09.623", "error": { "time": "2024-05-04T12:33:23.177302241+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449523794476531713, "checkpoint_time": "2024-05-04 12:33:09.623", "error": { "time": "2024-05-04T12:33:23.177302241+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449523794476531713, "checkpoint_time": "2024-05-04 12:33:09.623", "error": { "time": "2024-05-04T12:33:23.177302241+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449523794476531713, '"checkpoint_time":' '"2024-05-04' '12:33:09.623",' '"error":' '{' '"time":' '"2024-05-04T12:33:23.177302241+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449523794476531713, '"checkpoint_time":' '"2024-05-04' '12:33:09.623",' '"error":' '{' '"time":' '"2024-05-04T12:33:23.177302241+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5781.out cli changefeed remove -c changefeed-error go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da run task successfully go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 check_changefeed_state http://127.0.0.1:2379 42627c8f-363e-4b1e-b2ba-62a9055b9bd5 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=42627c8f-363e-4b1e-b2ba-62a9055b9bd5 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 42627c8f-363e-4b1e-b2ba-62a9055b9bd5 -s table ddl_manager.finish_mark not exists for 31-th check, retry later Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449523794476531713 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS + info='{ "upstream_id": 7364997798452918721, "namespace": "default", "id": "42627c8f-363e-4b1e-b2ba-62a9055b9bd5", "state": "normal", "checkpoint_tso": 449523797047902211, "checkpoint_time": "2024-05-04 12:33:19.432", "error": null }' + echo '{ "upstream_id": 7364997798452918721, "namespace": "default", "id": "42627c8f-363e-4b1e-b2ba-62a9055b9bd5", "state": "normal", "checkpoint_tso": 449523797047902211, "checkpoint_time": "2024-05-04 12:33:19.432", "error": null }' { "upstream_id": 7364997798452918721, "namespace": "default", "id": "42627c8f-363e-4b1e-b2ba-62a9055b9bd5", "state": "normal", "checkpoint_tso": 449523797047902211, "checkpoint_time": "2024-05-04 12:33:19.432", "error": null } ++ echo '{' '"upstream_id":' 7364997798452918721, '"namespace":' '"default",' '"id":' '"42627c8f-363e-4b1e-b2ba-62a9055b9bd5",' '"state":' '"normal",' '"checkpoint_tso":' 449523797047902211, '"checkpoint_time":' '"2024-05-04' '12:33:19.432",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364997798452918721, '"namespace":' '"default",' '"id":' '"42627c8f-363e-4b1e-b2ba-62a9055b9bd5",' '"state":' '"normal",' '"checkpoint_tso":' 449523797047902211, '"checkpoint_time":' '"2024-05-04' '12:33:19.432",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7364997798452918721 449523797047902210 449523797047902211 coverage: 2.0% of statements in github.com/pingcap/tiflow/... go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading k8s.io/apimachinery v0.28.6 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading golang.org/x/sync v0.7.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a \033[0;36m<<< Run all test success >>>\033[0m go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.8089.out cli tso query --pd=http://127.0.0.1:2379 go: downloading github.com/gorilla/mux v1.8.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading cloud.google.com/go v0.112.2 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/google/btree v1.1.2 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 run task successfully check_changefeed_state http://127.0.0.1:2379 42627c8f-363e-4b1e-b2ba-62a9055b9bd5 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=42627c8f-363e-4b1e-b2ba-62a9055b9bd5 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 42627c8f-363e-4b1e-b2ba-62a9055b9bd5 -s go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully + info='{ "upstream_id": 7364997798452918721, "namespace": "default", "id": "42627c8f-363e-4b1e-b2ba-62a9055b9bd5", "state": "stopped", "checkpoint_tso": 449523797047902211, "checkpoint_time": "2024-05-04 12:33:19.432", "error": null }' + echo '{ "upstream_id": 7364997798452918721, "namespace": "default", "id": "42627c8f-363e-4b1e-b2ba-62a9055b9bd5", "state": "stopped", "checkpoint_tso": 449523797047902211, "checkpoint_time": "2024-05-04 12:33:19.432", "error": null }' { "upstream_id": 7364997798452918721, "namespace": "default", "id": "42627c8f-363e-4b1e-b2ba-62a9055b9bd5", "state": "stopped", "checkpoint_tso": 449523797047902211, "checkpoint_time": "2024-05-04 12:33:19.432", "error": null } ++ echo '{' '"upstream_id":' 7364997798452918721, '"namespace":' '"default",' '"id":' '"42627c8f-363e-4b1e-b2ba-62a9055b9bd5",' '"state":' '"stopped",' '"checkpoint_tso":' 449523797047902211, '"checkpoint_time":' '"2024-05-04' '12:33:19.432",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7364997798452918721, '"namespace":' '"default",' '"id":' '"42627c8f-363e-4b1e-b2ba-62a9055b9bd5",' '"state":' '"stopped",' '"checkpoint_tso":' 449523797047902211, '"checkpoint_time":' '"2024-05-04' '12:33:19.432",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 76810f5a-0c89-4495-9806-0153351e9813 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=76810f5a-0c89-4495-9806-0153351e9813 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 76810f5a-0c89-4495-9806-0153351e9813 -s go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 table ddl_manager.finish_mark not exists for 32-th check, retry later + info='{ "upstream_id": 7364997798452918721, "namespace": "default", "id": "76810f5a-0c89-4495-9806-0153351e9813", "state": "normal", "checkpoint_tso": 449523799367876612, "checkpoint_time": "2024-05-04 12:33:28.282", "error": null }' + echo '{ "upstream_id": 7364997798452918721, "namespace": "default", "id": "76810f5a-0c89-4495-9806-0153351e9813", "state": "normal", "checkpoint_tso": 449523799367876612, "checkpoint_time": "2024-05-04 12:33:28.282", "error": null }' { "upstream_id": 7364997798452918721, "namespace": "default", "id": "76810f5a-0c89-4495-9806-0153351e9813", "state": "normal", "checkpoint_tso": 449523799367876612, "checkpoint_time": "2024-05-04 12:33:28.282", "error": null } ++ echo '{' '"upstream_id":' 7364997798452918721, '"namespace":' '"default",' '"id":' '"76810f5a-0c89-4495-9806-0153351e9813",' '"state":' '"normal",' '"checkpoint_tso":' 449523799367876612, '"checkpoint_time":' '"2024-05-04' '12:33:28.282",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364997798452918721, '"namespace":' '"default",' '"id":' '"76810f5a-0c89-4495-9806-0153351e9813",' '"state":' '"normal",' '"checkpoint_tso":' 449523799367876612, '"checkpoint_time":' '"2024-05-04' '12:33:28.282",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7364997798452918721 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da wait process cdc.test exit for 1-th time... go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 + set +x + tso='449523799208755201 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523799208755201 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:33:29 CST 2024] <<<<<< START cdc server in syncpoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.81148116.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 16.67 secs (223804153 bytes/sec) [Pipeline] { [Pipeline] cache go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 12:33:29 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.58805882.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 33-th check, retry later go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... run task successfully Changefeed remove successfully. ID: 42627c8f-363e-4b1e-b2ba-62a9055b9bd5 CheckpointTs: 449523797047902211 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_forward http://127.0.0.1:2379 7364997798452918721 449523800193892356 449523797047902211 449523799367876612 table ddl_manager.finish_mark not exists for 34-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:32 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1018984e-dd56-474f-97bc-863a879d04ac {"id":"1018984e-dd56-474f-97bc-863a879d04ac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797209} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e0a41de5 1018984e-dd56-474f-97bc-863a879d04ac /tidb/cdc/default/default/upstream/7364997866768255751 {"id":7364997866768255751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1018984e-dd56-474f-97bc-863a879d04ac {"id":"1018984e-dd56-474f-97bc-863a879d04ac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797209} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e0a41de5 1018984e-dd56-474f-97bc-863a879d04ac /tidb/cdc/default/default/upstream/7364997866768255751 {"id":7364997866768255751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1018984e-dd56-474f-97bc-863a879d04ac {"id":"1018984e-dd56-474f-97bc-863a879d04ac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797209} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e0a41de5 1018984e-dd56-474f-97bc-863a879d04ac /tidb/cdc/default/default/upstream/7364997866768255751 {"id":7364997866768255751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.8170.out cli changefeed create --start-ts=449523799208755201 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml Create changefeed successfully! ID: 74f034b2-8718-437c-af43-ff09fc45fa53 Info: {"upstream_id":7364997866768255751,"namespace":"default","id":"74f034b2-8718-437c-af43-ff09fc45fa53","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T12:33:32.677206758+08:00","start_ts":449523799208755201,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523799208755201,"checkpoint_ts":449523799208755201,"checkpoint_time":"2024-05-04 12:33:27.675"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:32 GMT < Content-Length: 877 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a452c74-7a96-41f0-b11f-37c7cceddf70 {"id":"3a452c74-7a96-41f0-b11f-37c7cceddf70","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797210} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023fe3c 3a452c74-7a96-41f0-b11f-37c7cceddf70 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a452c74-7a96-41f0-b11f-37c7cceddf70 {"id":"3a452c74-7a96-41f0-b11f-37c7cceddf70","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797210} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023fe3c 3a452c74-7a96-41f0-b11f-37c7cceddf70 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a452c74-7a96-41f0-b11f-37c7cceddf70 {"id":"3a452c74-7a96-41f0-b11f-37c7cceddf70","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797210} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023fe3c 3a452c74-7a96-41f0-b11f-37c7cceddf70 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5934.out cli changefeed create --start-ts=449523790963802113 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-1 run task successfully Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7364997725240484907,"namespace":"default","id":"changefeed-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T12:33:33.368266021+08:00","start_ts":449523790963802113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523790963802113,"checkpoint_ts":449523790963802113,"checkpoint_time":"2024-05-04 12:32:56.223"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Changefeed remove successfully. ID: 76810f5a-0c89-4495-9806-0153351e9813 CheckpointTs: 449523800456036359 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_cleared http://127.0.0.1:2379 7364997798452918721 run task successfully table ddl_manager.finish_mark not exists for 35-th check, retry later wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x ***************** properties ***************** "mysql.db"="testSync" "scanproportion"="0" "readallfields"="true" "workload"="core" "mysql.host"="127.0.0.1" "updateproportion"="0" "mysql.user"="root" "mysql.port"="4000" "dotransactions"="false" "insertproportion"="0" "readproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "threadcount"="2" "recordcount"="10" ********************************************** Run finished, takes 22.14704ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1630.9, Avg(us): 4336, Min(us): 1171, Max(us): 15934, 95th(us): 16000, 99th(us): 16000 wait process cdc.test exit for 2-th time... + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:33:35 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 1862 0 --:--:-- --:--:-- --:--:-- 1874 + info='{"state":"warning","resolved_ts":449523791357280258,"checkpoint_ts":449523791357280258,"last_warning":{"time":"2024-05-04T12:33:34.846412979+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449523791357280258,"checkpoint_ts":449523791357280258,"last_warning":{"time":"2024-05-04T12:33:34.846412979+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449523791357280258,"checkpoint_ts":449523791357280258,"last_warning":{"time":"2024-05-04T12:33:34.846412979+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ jq -r .state ++ echo '{"state":"warning","resolved_ts":449523791357280258,"checkpoint_ts":449523791357280258,"last_warning":{"time":"2024-05-04T12:33:34.846412979+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":449523791357280258,"checkpoint_ts":449523791357280258,"last_warning":{"time":"2024-05-04T12:33:34.846412979+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5972.out cli changefeed remove -c changefeed-error-1 Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449523791357280258 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 36-th check, retry later + set +x wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 37-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 12:33:39 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.60416043.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 38-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 39-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:42 GMT < Content-Length: 877 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25105558-3c4d-4f1d-b03e-964d8366eed5 {"id":"25105558-3c4d-4f1d-b03e-964d8366eed5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797219} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023febb 25105558-3c4d-4f1d-b03e-964d8366eed5 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25105558-3c4d-4f1d-b03e-964d8366eed5 {"id":"25105558-3c4d-4f1d-b03e-964d8366eed5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797219} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023febb 25105558-3c4d-4f1d-b03e-964d8366eed5 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25105558-3c4d-4f1d-b03e-964d8366eed5 {"id":"25105558-3c4d-4f1d-b03e-964d8366eed5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797219} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023febb 25105558-3c4d-4f1d-b03e-964d8366eed5 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6094.out cli changefeed create --start-ts=449523790963802113 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7364997725240484907,"namespace":"default","id":"changefeed-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T12:33:42.957337088+08:00","start_ts":449523790963802113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523790963802113,"checkpoint_ts":449523790963802113,"checkpoint_time":"2024-05-04 12:32:56.223"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 40-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449523790963802113, "checkpoint_time": "2024-05-04 12:32:56.223", "error": { "time": "2024-05-04T12:33:43.033552591+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449523790963802113 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449523790963802113, "checkpoint_time": "2024-05-04 12:32:56.223", "error": { "time": "2024-05-04T12:33:43.033552591+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449523790963802113 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449523790963802113, "checkpoint_time": "2024-05-04 12:32:56.223", "error": { "time": "2024-05-04T12:33:43.033552591+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449523790963802113 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449523790963802113, '"checkpoint_time":' '"2024-05-04' '12:32:56.223",' '"error":' '{' '"time":' '"2024-05-04T12:33:43.033552591+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449523790963802113 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449523790963802113, '"checkpoint_time":' '"2024-05-04' '12:32:56.223",' '"error":' '{' '"time":' '"2024-05-04T12:33:43.033552591+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449523790963802113 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449523790963802113 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449523790963802113 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6166.out cli changefeed remove -c changefeed-error-2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449523790963802113 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 41-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078497e40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:4030, start at 2024-05-04 12:33:45.503283094 +0800 CST m=+5.425173765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:45.512 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:45.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:45.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078497e40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:4030, start at 2024-05-04 12:33:45.503283094 +0800 CST m=+5.425173765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:45.512 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:45.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:45.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078499600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:4113, start at 2024-05-04 12:33:45.58028623 +0800 CST m=+5.437153614 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:45.589 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:45.560 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:45.560 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:33:48 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.62096211.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 42-th check, retry later table ddl_manager.finish_mark not exists for 43-th check, retry later [Sat May 4 12:33:50 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.53415343.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:51 GMT < Content-Length: 877 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f87315c-abd4-467d-a8ea-05594ab3d825 {"id":"4f87315c-abd4-467d-a8ea-05594ab3d825","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797228} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023ff20 4f87315c-abd4-467d-a8ea-05594ab3d825 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f87315c-abd4-467d-a8ea-05594ab3d825 {"id":"4f87315c-abd4-467d-a8ea-05594ab3d825","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797228} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023ff20 4f87315c-abd4-467d-a8ea-05594ab3d825 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f87315c-abd4-467d-a8ea-05594ab3d825 {"id":"4f87315c-abd4-467d-a8ea-05594ab3d825","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797228} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f41e023ff20 4f87315c-abd4-467d-a8ea-05594ab3d825 /tidb/cdc/default/default/upstream/7364997725240484907 {"id":7364997725240484907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6264.out cli changefeed create --start-ts=0 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-initialize-error Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7364997725240484907,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T12:33:52.090283006+08:00","start_ts":449523805565485061,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523805565485061,"checkpoint_ts":449523805565485061,"checkpoint_time":"2024-05-04 12:33:51.924"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 44-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:33:53 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e67a8e2-9300-41fc-a3cd-fa291f57fcc2 {"id":"3e67a8e2-9300-41fc-a3cd-fa291f57fcc2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797230} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e0f908cc 3e67a8e2-9300-41fc-a3cd-fa291f57fcc2 /tidb/cdc/default/default/upstream/7364997965301807933 {"id":7364997965301807933,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e67a8e2-9300-41fc-a3cd-fa291f57fcc2 {"id":"3e67a8e2-9300-41fc-a3cd-fa291f57fcc2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797230} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e0f908cc 3e67a8e2-9300-41fc-a3cd-fa291f57fcc2 /tidb/cdc/default/default/upstream/7364997965301807933 {"id":7364997965301807933,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e67a8e2-9300-41fc-a3cd-fa291f57fcc2 {"id":"3e67a8e2-9300-41fc-a3cd-fa291f57fcc2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797230} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e0f908cc 3e67a8e2-9300-41fc-a3cd-fa291f57fcc2 /tidb/cdc/default/default/upstream/7364997965301807933 {"id":7364997965301807933,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... + info='{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449523805565485061, "checkpoint_time": "2024-05-04 12:33:51.924", "error": { "time": "2024-05-04T12:33:52.268772544+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449523805565485061, "checkpoint_time": "2024-05-04 12:33:51.924", "error": { "time": "2024-05-04T12:33:52.268772544+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449523805565485061, "checkpoint_time": "2024-05-04 12:33:51.924", "error": { "time": "2024-05-04T12:33:52.268772544+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449523805565485061, '"checkpoint_time":' '"2024-05-04' '12:33:51.924",' '"error":' '{' '"time":' '"2024-05-04T12:33:52.268772544+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later ***************** properties ***************** "operationcount"="0" "dotransactions"="false" "recordcount"="50" "updateproportion"="0" "readallfields"="true" "mysql.db"="changefeed_reconstruct" "mysql.host"="127.0.0.1" "workload"="core" "readproportion"="0" "mysql.user"="root" "requestdistribution"="uniform" "insertproportion"="0" "threadcount"="4" "scanproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 18.520784ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3421.7, Avg(us): 1484, Min(us): 1010, Max(us): 4431, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 45-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 46-th check, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449523805565485061, "checkpoint_time": "2024-05-04 12:33:51.924", "error": { "time": "2024-05-04T12:33:52.268772544+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449523805565485061, "checkpoint_time": "2024-05-04 12:33:51.924", "error": { "time": "2024-05-04T12:33:52.268772544+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449523805565485061, "checkpoint_time": "2024-05-04 12:33:51.924", "error": { "time": "2024-05-04T12:33:52.268772544+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449523805565485061, '"checkpoint_time":' '"2024-05-04' '12:33:51.924",' '"error":' '{' '"time":' '"2024-05-04T12:33:52.268772544+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 24.46 secs (152513246 bytes/sec) [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] sh [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G18 Run cases: availability http_proxies sequence PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=cb74bcbe-9c7b-4060-b40d-6eefae5f9ff3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G18 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-2nt2l GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-2nt2l pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/availability/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G21 Run cases: bank kill_owner_with_ddl owner_remove_table_error PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=2f9fb84f-0ecc-4cd6-9f75-1933d385b4b2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G21 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-dm64w GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj pingcap_tiflow_pull_cdc_integration_test_1731-dm64w GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 47-th check, retry later wait process 5346 exit for 1-th time... wait process 5346 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5346) - No such process wait process 5346 exit for 3-th time... process 5346 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sat May 4 12:33:59 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.55885590.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G14 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=6152df22-5412-4e88-81f0-35e3b6a493ab BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-x4lzj GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-x4lzj pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449523807596838913, "checkpoint_time": "2024-05-04 12:33:59.673", "error": null }' + echo '{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449523807596838913, "checkpoint_time": "2024-05-04 12:33:59.673", "error": null }' { "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449523807596838913, "checkpoint_time": "2024-05-04 12:33:59.673", "error": null } ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449523807596838913, '"checkpoint_time":' '"2024-05-04' '12:33:59.673",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449523807596838913, '"checkpoint_time":' '"2024-05-04' '12:33:59.673",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6438.out cli changefeed pause -c changefeed-initialize-error table ddl_manager.finish_mark not exists for 48-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff/output" source-instances = ["tidb"] target-instance = "mysql" target-check-tables = ["consistent_replicate_ddl.usertable*","consistent_replicate_ddl.check*"] [data-sources] [data-sources.tidb] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449523772619227157" [data-sources.mysql] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/04 12:33:49.008 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/04 12:33:49.009 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/04 12:33:49.009 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/04 12:33:49.009 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/04 12:33:49.009 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=449523770954874894] [resolvedTs=449523772619227157] [2024/05/04 12:33:49.011 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 12:33:49.011 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 12:33:49.011 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 12:33:49.012 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=2.972549ms] [2024/05/04 12:33:49.012 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=3.116944ms] [2024/05/04 12:33:49.016 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 12:33:49.018 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/04 12:33:49.019 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.019 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/04 12:33:49.020 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/04 12:33:49.020 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 12:33:49.020 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 12:33:49.020 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 12:33:49.023 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 12:33:49.025 +08:00] [INFO] [mysql_ddl_sink.go:107] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/04 12:33:49.025 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523770954874888,\"CommitTs\":449523770954874894,\"Query\":\"DROP TABLE `consistent_replicate_ddl`.`usertable1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":108,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":4,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:49.026 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [commitTs=449523770954874894] [namespace=] [changefeed=] [2024/05/04 12:33:49.231 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [duration=205.258258ms] [namespace=] [changefeed=] [2024/05/04 12:33:49.231 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523771072577546,\"CommitTs\":449523771085946882,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":114,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:49.231 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [commitTs=449523771085946882] [namespace=] [changefeed=] [2024/05/04 12:33:49.373 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [duration=141.986297ms] [namespace=] [changefeed=] [2024/05/04 12:33:49.374 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523771217018890,\"CommitTs\":449523771230126082,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1_1\",\"TableID\":114,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:49.574 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [commitTs=449523771230126082] [namespace=] [changefeed=] [2024/05/04 12:33:49.717 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [duration=143.507458ms] [namespace=] [changefeed=] [2024/05/04 12:33:49.718 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523771347828764,\"CommitTs\":449523771347828767,\"Query\":\"CREATE DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":1,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:49.918 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="CREATE DATABASE consistent_replicate_ddl1"] [commitTs=449523771347828767] [namespace=] [changefeed=] [2024/05/04 12:33:49.997 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="CREATE DATABASE consistent_replicate_ddl1"] [duration=78.988731ms] [namespace=] [changefeed=] [2024/05/04 12:33:49.997 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523771492007953,\"CommitTs\":449523771492007956,\"Query\":\"ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":26,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:49.997 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [commitTs=449523771492007956] [namespace=] [changefeed=] [2024/05/04 12:33:50.071 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=73.815748ms] [namespace=] [changefeed=] [2024/05/04 12:33:50.071 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523771623342107,\"CommitTs\":449523771623342110,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:50.071 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [commitTs=449523771623342110] [namespace=] [changefeed=] [2024/05/04 12:33:50.146 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=74.960664ms] [namespace=] [changefeed=] [2024/05/04 12:33:50.147 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523771819950088,\"CommitTs\":449523771819950092,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":6,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:50.348 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [commitTs=449523771819950092] [namespace=] [changefeed=] [2024/05/04 12:33:50.612 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [duration=263.86016ms] [namespace=] [changefeed=] [2024/05/04 12:33:50.612 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523771964129286,\"CommitTs\":449523771964129291,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":5,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:50.612 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [commitTs=449523771964129291] [namespace=] [changefeed=] [2024/05/04 12:33:50.874 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [duration=261.957378ms] [namespace=] [changefeed=] [2024/05/04 12:33:50.874 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523772082094088,\"CommitTs\":449523772082094091,\"Query\":\"DROP DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":2,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:50.874 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="DROP DATABASE consistent_replicate_ddl1"] [commitTs=449523772082094091] [namespace=] [changefeed=] [2024/05/04 12:33:51.071 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="DROP DATABASE consistent_replicate_ddl1"] [duration=196.856455ms] [namespace=] [changefeed=] [2024/05/04 12:33:51.071 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523772186951704,\"CommitTs\":449523772186951707,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:51.071 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [commitTs=449523772186951707] [namespace=] [changefeed=] [2024/05/04 12:33:51.146 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [duration=74.977154ms] [namespace=] [changefeed=] [2024/05/04 12:33:51.146 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523772343975943,\"CommitTs\":449523772343975948,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2_1\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:51.146 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [commitTs=449523772343975948] [namespace=] [changefeed=] [2024/05/04 12:33:51.281 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [duration=135.505486ms] [namespace=] [changefeed=] [2024/05/04 12:33:51.282 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523772343975943,\"CommitTs\":449523772343975948,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:51.282 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [commitTs=449523772343975948] [namespace=] [changefeed=] [2024/05/04 12:33:51.421 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [duration=139.782012ms] [namespace=] [changefeed=] [2024/05/04 12:33:51.421 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523772462202904,\"CommitTs\":449523772462202908,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":112,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":12,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:51.422 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [commitTs=449523772462202908] [namespace=] [changefeed=] [2024/05/04 12:33:51.499 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [duration=77.62874ms] [namespace=] [changefeed=] [2024/05/04 12:33:51.499 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523772619227152,\"CommitTs\":449523772619227157,\"Query\":\"CREATE table consistent_replicate_ddl.check1(id int primary key)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"check1\",\"TableID\":129,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:33:51.700 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [commitTs=449523772619227157] [namespace=] [changefeed=] [2024/05/04 12:33:51.779 +08:00] [INFO] [mysql_ddl_sink.go:241] ["Exec DDL succeeded"] [sql="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [duration=78.967166ms] [namespace=] [changefeed=] [2024/05/04 12:33:51.779 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}] [checkpointTs=449523772619227157] [2024/05/04 12:33:51.779 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}] [checkpointTs=449523772619227157] [2024/05/04 12:33:51.879 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449523772619227157] [2024/05/04 12:33:51.879 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449523772619227157] [2024/05/04 12:33:51.980 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449523772619227157] [2024/05/04 12:33:51.980 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449523772619227157] [2024/05/04 12:33:52.080 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=101] [appliedDDLCount=14] [currentCheckpoint=449523772619227157] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/04 12:33:52.080 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] Apply redo log successfully table consistent_replicate_ddl.check1 exists check diff successfully [Sat May 4 12:33:52 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> count(*) 50 start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078570d0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2, pid:4625, start at 2024-05-04 12:33:59.36228811 +0800 CST m=+5.474533921 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:59.368 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:59.348 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:59.348 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078570d0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2, pid:4625, start at 2024-05-04 12:33:59.36228811 +0800 CST m=+5.474533921 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:59.368 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:59.348 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:59.348 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078570b0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-d59k5-vscx3-4f9l2, pid:4709, start at 2024-05-04 12:33:59.354923938 +0800 CST m=+5.394784411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:35:59.362 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:33:59.340 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:23:59.340 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend table ddl_manager.finish_mark not exists for 49-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449523807596838913, "checkpoint_time": "2024-05-04 12:33:59.673", "error": { "time": "2024-05-04T12:33:52.268772544+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449523807596838913, "checkpoint_time": "2024-05-04 12:33:59.673", "error": { "time": "2024-05-04T12:33:52.268772544+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449523807596838913, "checkpoint_time": "2024-05-04 12:33:59.673", "error": { "time": "2024-05-04T12:33:52.268772544+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449523807596838913, '"checkpoint_time":' '"2024-05-04' '12:33:59.673",' '"error":' '{' '"time":' '"2024-05-04T12:33:52.268772544+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449523807596838913, '"checkpoint_time":' '"2024-05-04' '12:33:59.673",' '"error":' '{' '"time":' '"2024-05-04T12:33:52.268772544+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6517.out cli changefeed resume -c changefeed-initialize-error Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {UpstreamID:7364997965301807933 Namespace:default ID:a79a54d8-ccad-40dd-8af9-d332a3d7d1fc SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-04 12:33:53.694140814 +0800 CST StartTs:449523805995663365 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024f4ab0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523806021877765} {CheckpointTs:449523808237256710 MinTableBarrierTs:449523808237256710 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523808237256710, checkpointTs: 449523808237256710, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be4cd348-55e5-447c-9ddb-6da6fe673804 {"id":"be4cd348-55e5-447c-9ddb-6da6fe673804","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797240} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e0f9096c be4cd348-55e5-447c-9ddb-6da6fe673804 /tidb/cdc/default/default/changefeed/info/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {"upstream-id":7364997965301807933,"namespace":"default","changefeed-id":"a79a54d8-ccad-40dd-8af9-d332a3d7d1fc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T12:33:53.694140814+08:00","start-ts":449523805995663365,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523806021877765} /tidb/cdc/default/default/changefeed/status/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {"checkpoint-ts":449523808237256710,"min-table-barrier-ts":449523808237256710,"admin-job-type":0} /tidb/cdc/default/default/task/position/be4cd348-55e5-447c-9ddb-6da6fe673804/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997965301807933 {"id":7364997965301807933,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {UpstreamID:7364997965301807933 Namespace:default ID:a79a54d8-ccad-40dd-8af9-d332a3d7d1fc SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-04 12:33:53.694140814 +0800 CST StartTs:449523805995663365 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024f4ab0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523806021877765} {CheckpointTs:449523808237256710 MinTableBarrierTs:449523808237256710 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523808237256710, checkpointTs: 449523808237256710, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be4cd348-55e5-447c-9ddb-6da6fe673804 {"id":"be4cd348-55e5-447c-9ddb-6da6fe673804","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797240} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e0f9096c be4cd348-55e5-447c-9ddb-6da6fe673804 /tidb/cdc/default/default/changefeed/info/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {"upstream-id":7364997965301807933,"namespace":"default","changefeed-id":"a79a54d8-ccad-40dd-8af9-d332a3d7d1fc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T12:33:53.694140814+08:00","start-ts":449523805995663365,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523806021877765} /tidb/cdc/default/default/changefeed/status/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {"checkpoint-ts":449523808237256710,"min-table-barrier-ts":449523808237256710,"admin-job-type":0} /tidb/cdc/default/default/task/position/be4cd348-55e5-447c-9ddb-6da6fe673804/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997965301807933 {"id":7364997965301807933,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {UpstreamID:7364997965301807933 Namespace:default ID:a79a54d8-ccad-40dd-8af9-d332a3d7d1fc SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-04 12:33:53.694140814 +0800 CST StartTs:449523805995663365 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0024f4ab0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523806021877765} {CheckpointTs:449523808237256710 MinTableBarrierTs:449523808237256710 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523808237256710, checkpointTs: 449523808237256710, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be4cd348-55e5-447c-9ddb-6da6fe673804 {"id":"be4cd348-55e5-447c-9ddb-6da6fe673804","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797240} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e0f9096c be4cd348-55e5-447c-9ddb-6da6fe673804 /tidb/cdc/default/default/changefeed/info/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {"upstream-id":7364997965301807933,"namespace":"default","changefeed-id":"a79a54d8-ccad-40dd-8af9-d332a3d7d1fc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T12:33:53.694140814+08:00","start-ts":449523805995663365,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523806021877765} /tidb/cdc/default/default/changefeed/status/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {"checkpoint-ts":449523808237256710,"min-table-barrier-ts":449523808237256710,"admin-job-type":0} /tidb/cdc/default/default/task/position/be4cd348-55e5-447c-9ddb-6da6fe673804/a79a54d8-ccad-40dd-8af9-d332a3d7d1fc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364997965301807933 {"id":7364997965301807933,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC2F0201539107 < X-Xss-Protection: 1; mode=block < Date: Sat, 04 May 2024 04:34:03 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... [Sat May 4 12:34:04 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.60186020.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 "id": "be4cd348-55e5-447c-9ddb-6da6fe673804", "cluster-id": "default" run task successfully + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s table ddl_manager.finish_mark not exists for 50-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... + info='{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449523807596838913, "checkpoint_time": "2024-05-04 12:33:59.673", "error": null }' + echo '{ "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449523807596838913, "checkpoint_time": "2024-05-04 12:33:59.673", "error": null }' { "upstream_id": 7364997725240484907, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449523807596838913, "checkpoint_time": "2024-05-04 12:33:59.673", "error": null } ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449523807596838913, '"checkpoint_time":' '"2024-05-04' '12:33:59.673",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364997725240484907, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449523807596838913, '"checkpoint_time":' '"2024-05-04' '12:33:59.673",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6606.out cli changefeed remove -c changefeed-initialize-error Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449523807596838913 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... capture_id: be4cd348-55e5-447c-9ddb-6da6fe673804 check_processor_table_count http://127.0.0.1:2379 a79a54d8-ccad-40dd-8af9-d332a3d7d1fc be4cd348-55e5-447c-9ddb-6da6fe673804 1 run task successfully check_processor_table_count http://127.0.0.1:2379 a79a54d8-ccad-40dd-8af9-d332a3d7d1fc be4cd348-55e5-447c-9ddb-6da6fe673804 0 table ddl_manager.finish_mark not exists for 51-th check, retry later table count 1 does equal to expected count 0 run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Verifying downstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 6.94 secs (537311563 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:07 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e6265acf-2af0-4bbf-b9c8-5938f9fe2be4 {"id":"e6265acf-2af0-4bbf-b9c8-5938f9fe2be4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797244} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e13106cd e6265acf-2af0-4bbf-b9c8-5938f9fe2be4 /tidb/cdc/default/default/upstream/7364998022843701722 {"id":7364998022843701722,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e6265acf-2af0-4bbf-b9c8-5938f9fe2be4 {"id":"e6265acf-2af0-4bbf-b9c8-5938f9fe2be4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797244} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e13106cd e6265acf-2af0-4bbf-b9c8-5938f9fe2be4 /tidb/cdc/default/default/upstream/7364998022843701722 {"id":7364998022843701722,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e6265acf-2af0-4bbf-b9c8-5938f9fe2be4 {"id":"e6265acf-2af0-4bbf-b9c8-5938f9fe2be4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797244} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e13106cd e6265acf-2af0-4bbf-b9c8-5938f9fe2be4 /tidb/cdc/default/default/upstream/7364998022843701722 {"id":7364998022843701722,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 52-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:34:08 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> check_processor_table_count http://127.0.0.1:2379 a79a54d8-ccad-40dd-8af9-d332a3d7d1fc be4cd348-55e5-447c-9ddb-6da6fe673804 0 run task successfully ***************** properties ***************** "workload"="core" "mysql.db"="changefeed_reconstruct" "mysql.port"="4000" "threadcount"="4" "readproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "recordcount"="50" "requestdistribution"="uniform" "readallfields"="true" "updateproportion"="0" "mysql.user"="root" "dotransactions"="false" "scanproportion"="0" "insertproportion"="0" ********************************************** Run finished, takes 20.400209ms INSERT - Takes(s): 0.0, Count: 48, OPS: 2921.7, Avg(us): 1588, Min(us): 931, Max(us): 4259, 95th(us): 5000, 99th(us): 5000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable not exists for 1-th check, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:34:06 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 53-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable exists check diff successfully Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 54-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:34:12 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078632680017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj, pid:1282, start at 2024-05-04 12:34:11.758571327 +0800 CST m=+5.180927500 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:11.765 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:11.738 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:11.738 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078632680017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj, pid:1282, start at 2024-05-04 12:34:11.758571327 +0800 CST m=+5.180927500 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:11.765 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:11.738 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:11.738 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0786331c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj, pid:1361, start at 2024-05-04 12:34:11.790791373 +0800 CST m=+5.152191519 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:11.797 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:11.783 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:11.783 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bank/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bank/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bank/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bank/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bank/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078642300010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w, pid:1298, start at 2024-05-04 12:34:12.763524971 +0800 CST m=+5.334319728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:12.770 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:12.748 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:12.748 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_pause_resume.t3 exists Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 55-th check, retry later [Sat May 4 12:34:15 CST 2024] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.26142616.out server --log-file /tmp/tidb_cdc_test/bank/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078642300010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w, pid:1298, start at 2024-05-04 12:34:12.763524971 +0800 CST m=+5.334319728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:12.770 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:12.748 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:12.748 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0786439c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-2nt2l-3g3s0-5w26w, pid:1377, start at 2024-05-04 12:34:12.85350189 +0800 CST m=+5.342578421 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:12.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:12.839 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:12.839 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/availability/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/availability/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/availability/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/availability/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/availability/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078671c80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp, pid:1235, start at 2024-05-04 12:34:15.824777163 +0800 CST m=+5.618795913 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:15.832 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:15.794 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:15.794 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 56-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2638.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:18 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/23d83053-d695-40f5-b849-cf72995a93b2 {"id":"23d83053-d695-40f5-b849-cf72995a93b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797255} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1648ad4 23d83053-d695-40f5-b849-cf72995a93b2 /tidb/cdc/default/default/upstream/7364998081617273337 {"id":7364998081617273337,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/23d83053-d695-40f5-b849-cf72995a93b2 {"id":"23d83053-d695-40f5-b849-cf72995a93b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797255} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1648ad4 23d83053-d695-40f5-b849-cf72995a93b2 /tidb/cdc/default/default/upstream/7364998081617273337 {"id":7364998081617273337,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/23d83053-d695-40f5-b849-cf72995a93b2 {"id":"23d83053-d695-40f5-b849-cf72995a93b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797255} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1648ad4 23d83053-d695-40f5-b849-cf72995a93b2 /tidb/cdc/default/default/upstream/7364998081617273337 {"id":7364998081617273337,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.2665.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ table ddl_manager.finish_mark not exists for 57-th check, retry later Create changefeed successfully! ID: b309bafc-ee61-4180-8d1f-2be54f0d9e34 Info: {"upstream_id":7364998081617273337,"namespace":"default","id":"b309bafc-ee61-4180-8d1f-2be54f0d9e34","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-04T12:34:18.580276534+08:00","start_ts":449523812515971077,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523812515971077,"checkpoint_ts":449523812515971077,"checkpoint_time":"2024-05-04 12:34:18.438"} PASS =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078671c80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp, pid:1235, start at 2024-05-04 12:34:15.824777163 +0800 CST m=+5.618795913 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:15.832 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:15.794 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:15.794 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078672fc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp, pid:1321, start at 2024-05-04 12:34:15.910108981 +0800 CST m=+5.625546019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:15.918 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:15.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:15.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449523812426842113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523812426842113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later run test case test_kill_owner [Sat May 4 12:34:19 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.26992701.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + GO111MODULE=on + go run bank.go case.go -u 'root@tcp(127.0.0.1:4000)/bank' -d 'root@tcp(127.0.0.1:3306)/bank' --test-round=20000 -a 127.0.0.1:20080 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading google.golang.org/grpc v1.62.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.uber.org/multierr v1.11.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda [Sat May 4 12:34:20 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.26232625.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 58-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0786a68c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:4255, start at 2024-05-04 12:34:19.208812175 +0800 CST m=+5.420702706 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:19.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:19.221 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:19.221 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0786a68c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:4255, start at 2024-05-04 12:34:19.208812175 +0800 CST m=+5.420702706 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:19.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:19.221 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:19.221 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0786a97c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:4327, start at 2024-05-04 12:34:19.367623497 +0800 CST m=+5.527799384 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:19.377 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:19.359 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:19.359 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 59-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:22 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45c74a35-a890-4bd7-ac4c-0e2aa90505b5 {"id":"45c74a35-a890-4bd7-ac4c-0e2aa90505b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797260} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cdfe 45c74a35-a890-4bd7-ac4c-0e2aa90505b5 /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45c74a35-a890-4bd7-ac4c-0e2aa90505b5 {"id":"45c74a35-a890-4bd7-ac4c-0e2aa90505b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797260} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cdfe 45c74a35-a890-4bd7-ac4c-0e2aa90505b5 /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45c74a35-a890-4bd7-ac4c-0e2aa90505b5 {"id":"45c74a35-a890-4bd7-ac4c-0e2aa90505b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797260} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cdfe 45c74a35-a890-4bd7-ac4c-0e2aa90505b5 /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2757.out cli changefeed create --start-ts=449523812426842113 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 361cab25-2b13-4166-88e2-41e4aa8a9624 Info: {"upstream_id":7364998079142423954,"namespace":"default","id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:34:23.420688458+08:00","start_ts":449523812426842113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523812426842113,"checkpoint_ts":449523812426842113,"checkpoint_time":"2024-05-04 12:34:18.098"} PASS start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:23 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85f04ff4-58bd-4dc8-87a4-a7300162419d {"id":"85f04ff4-58bd-4dc8-87a4-a7300162419d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797261} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e16e84ce 85f04ff4-58bd-4dc8-87a4-a7300162419d /tidb/cdc/default/default/upstream/7364998093334425536 {"id":7364998093334425536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85f04ff4-58bd-4dc8-87a4-a7300162419d {"id":"85f04ff4-58bd-4dc8-87a4-a7300162419d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797261} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e16e84ce 85f04ff4-58bd-4dc8-87a4-a7300162419d /tidb/cdc/default/default/upstream/7364998093334425536 {"id":7364998093334425536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85f04ff4-58bd-4dc8-87a4-a7300162419d {"id":"85f04ff4-58bd-4dc8-87a4-a7300162419d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797261} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e16e84ce 85f04ff4-58bd-4dc8-87a4-a7300162419d /tidb/cdc/default/default/upstream/7364998093334425536 {"id":7364998093334425536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 12:34:24 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.55635565.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 60-th check, retry later Verifying downstream PD is started... + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 14.68 secs (254071064 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G20 Run cases: tidb_mysql_test ddl_reentrant multi_cdc_cluster PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=9ea64653-f447-4690-b7fa-122bf4ffacc0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G20 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-9570k GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-9570k pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<================= [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=a723736f-f388-474d-a241-e0cec0bd6f2c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-j6n6k GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-j6n6k pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 5-th time, retry later "is-owner": true, run task successfully table ddl_manager.finish_mark not exists for 61-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 2-th time, retry later Starting Upstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:27 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cd4388f-2515-43f2-b67f-e430de2688f6 {"id":"2cd4388f-2515-43f2-b67f-e430de2688f6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797264} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e17cb1d0 2cd4388f-2515-43f2-b67f-e430de2688f6 /tidb/cdc/default/default/upstream/7364998104416399251 {"id":7364998104416399251,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cd4388f-2515-43f2-b67f-e430de2688f6 {"id":"2cd4388f-2515-43f2-b67f-e430de2688f6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797264} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e17cb1d0 2cd4388f-2515-43f2-b67f-e430de2688f6 /tidb/cdc/default/default/upstream/7364998104416399251 {"id":7364998104416399251,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cd4388f-2515-43f2-b67f-e430de2688f6 {"id":"2cd4388f-2515-43f2-b67f-e430de2688f6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797264} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e17cb1d0 2cd4388f-2515-43f2-b67f-e430de2688f6 /tidb/cdc/default/default/upstream/7364998104416399251 {"id":7364998104416399251,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully owner pid: 2704 owner id 45c74a35-a890-4bd7-ac4c-0e2aa90505b5 [Sat May 4 12:34:27 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.28852887.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 62-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully Starting Upstream TiDB... table consistent_replicate_gbk.GBKTABLE not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 63-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... table consistent_replicate_gbk.GBKTABLE exists table consistent_replicate_gbk.check1 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00230d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523815939571721 MinTableBarrierTs:449523815939571721 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45c74a35-a890-4bd7-ac4c-0e2aa90505b5 {"id":"45c74a35-a890-4bd7-ac4c-0e2aa90505b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797260} /tidb/cdc/default/__cdc_meta__/capture/c1d08870-6ac1-4861-8131-15700138ee0a {"id":"c1d08870-6ac1-4861-8131-15700138ee0a","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797268} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cdfe 45c74a35-a890-4bd7-ac4c-0e2aa90505b5 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163ce69 c1d08870-6ac1-4861-8131-15700138ee0a /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523815939571721,"min-table-barrier-ts":449523815939571721,"admin-job-type":0} /tidb/cdc/default/default/task/position/45c74a35-a890-4bd7-ac4c-0e2aa90505b5/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c1d08870-6ac1-4861-8131-15700138ee0a/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00230d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523815939571721 MinTableBarrierTs:449523815939571721 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45c74a35-a890-4bd7-ac4c-0e2aa90505b5 {"id":"45c74a35-a890-4bd7-ac4c-0e2aa90505b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797260} /tidb/cdc/default/__cdc_meta__/capture/c1d08870-6ac1-4861-8131-15700138ee0a {"id":"c1d08870-6ac1-4861-8131-15700138ee0a","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797268} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cdfe 45c74a35-a890-4bd7-ac4c-0e2aa90505b5 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163ce69 c1d08870-6ac1-4861-8131-15700138ee0a /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523815939571721,"min-table-barrier-ts":449523815939571721,"admin-job-type":0} /tidb/cdc/default/default/task/position/45c74a35-a890-4bd7-ac4c-0e2aa90505b5/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/task/position/c1d08870-6ac1-4861-8131-15700138ee0a/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00230d9e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523815939571721 MinTableBarrierTs:449523815939571721 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45c74a35-a890-4bd7-ac4c-0e2aa90505b5 {"id":"45c74a35-a890-4bd7-ac4c-0e2aa90505b5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797260} /tidb/cdc/default/__cdc_meta__/capture/c1d08870-6ac1-4861-8131-15700138ee0a {"id":"c1d08870-6ac1-4861-8131-15700138ee0a","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797268} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cdfe 45c74a35-a890-4bd7-ac4c-0e2aa90505b5 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163ce69 c1d08870-6ac1-4861-8131-15700138ee0a /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523815939571721,"min-table-barrier-ts":449523815939571721,"admin-job-type":0} /tidb/cdc/default/default/task/position/45c74a35-a890-4bd7-ac4c-0e2aa90505b5/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c1d08870-6ac1-4861-8131-15700138ee0a/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "45c74a35-a890-4bd7-ac4c-0e2aa90505b5" | grep -v cluster_id | grep id [2024/05/04 12:34:30.914 +08:00] [INFO] [case.go:445] ["open db success"] [dsn=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:34:30.915 +08:00] [INFO] [case.go:445] ["open db success"] [dsn=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:34:30.915 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq0] [2024/05/04 12:34:31.014 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.39434ms"] [query="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/04 12:34:31.014 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq1] [2024/05/04 12:34:31.094 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.156558ms"] [query="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/04 12:34:31.094 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq2] [2024/05/04 12:34:31.174 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.124527ms"] [query="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/04 12:34:31.174 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq3] [2024/05/04 12:34:31.256 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.456806ms"] [query="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/04 12:34:31.256 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq4] [2024/05/04 12:34:31.337 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.959566ms"] [query="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/04 12:34:31.338 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts0] [2024/05/04 12:34:31.419 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.829458ms"] [query="INSERT IGNORE INTO accounts0 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/04 12:34:31.419 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts1] [2024/05/04 12:34:31.502 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.711331ms"] [query="INSERT IGNORE INTO accounts1 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/04 12:34:31.502 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts2] [2024/05/04 12:34:31.582 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.56395ms"] [query="INSERT IGNORE INTO accounts2 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/04 12:34:31.582 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts3] [2024/05/04 12:34:31.663 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.067892ms"] [query="INSERT IGNORE INTO accounts3 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/04 12:34:31.663 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts4] [2024/05/04 12:34:31.741 +08:00] [INFO] [case.go:365] ["insert 100 takes 4.324985ms"] [query="INSERT IGNORE INTO accounts4 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] table ddl_manager.finish_mark not exists for 64-th check, retry later table consistent_replicate_gbk.check1 exists check diff successfully wait process cdc.test exit for 1-th time... "id": "c1d08870-6ac1-4861-8131-15700138ee0a", "cluster-id": "default" "cluster-id": "default" run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078779540017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:7579, start at 2024-05-04 12:34:32.682727225 +0800 CST m=+5.164379457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:32.690 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:32.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:32.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078779540017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:7579, start at 2024-05-04 12:34:32.682727225 +0800 CST m=+5.164379457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:32.690 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:32.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:32.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07877c600016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:7661, start at 2024-05-04 12:34:32.901516764 +0800 CST m=+5.330701819 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:32.911 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:32.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:32.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:34:33.817 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 2-th time, retry later wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:34:34 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.57425744.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 65-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07878b580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:6545, start at 2024-05-04 12:34:33.850790691 +0800 CST m=+5.462734087 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:33.857 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:33.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:33.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) capture_id: c1d08870-6ac1-4861-8131-15700138ee0a Verifying downstream PD is started... wait process 2704 exit for 1-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.8952.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07879bf00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:6828, start at 2024-05-04 12:34:34.89747339 +0800 CST m=+5.480244694 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:34.905 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:34.876 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:34.876 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:34:35.829 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] wait process 2704 exit for 2-th time... wait process 2704 exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 66-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07878b580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:6545, start at 2024-05-04 12:34:33.850790691 +0800 CST m=+5.462734087 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:33.857 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:33.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:33.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07878da00011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:6631, start at 2024-05-04 12:34:33.980568724 +0800 CST m=+5.536139291 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:33.989 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:33.960 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:33.960 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2704) - No such process wait process 2704 exit for 4-th time... process 2704 already exit cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep c1d08870-6ac1-4861-8131-15700138ee0a -A1 | grep '"is-owner": true' + set +x + tso='449523817161949185 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523817161949185 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:34:37 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.89958997.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07879bf00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:6828, start at 2024-05-04 12:34:34.89747339 +0800 CST m=+5.480244694 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:34.905 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:34.876 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:34.876 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07879bf80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:6881, start at 2024-05-04 12:34:34.897003176 +0800 CST m=+5.406823978 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:34.904 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:34.878 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:34.878 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {UpstreamID:7364998104416399251 Namespace:default ID:7ce3deb2-1cd2-4297-878c-0ac02d25e66f SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:27.374996846 +0800 CST StartTs:449523814831489026 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003502000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523814857703426} {CheckpointTs:449523817334964229 MinTableBarrierTs:449523817334964229 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523817334964229, checkpointTs: 449523817334964229, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523817334964229, checkpointTs: 449523817334964229, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8 {"id":"dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e17cb27c dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8 /tidb/cdc/default/default/changefeed/info/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {"upstream-id":7364998104416399251,"namespace":"default","changefeed-id":"7ce3deb2-1cd2-4297-878c-0ac02d25e66f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:27.374996846+08:00","start-ts":449523814831489026,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523814857703426} /tidb/cdc/default/default/changefeed/status/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {"checkpoint-ts":449523817334964229,"min-table-barrier-ts":449523817334964229,"admin-job-type":0} /tidb/cdc/default/default/task/position/dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998104416399251 {"id":7364998104416399251,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {UpstreamID:7364998104416399251 Namespace:default ID:7ce3deb2-1cd2-4297-878c-0ac02d25e66f SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:27.374996846 +0800 CST StartTs:449523814831489026 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003502000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523814857703426} {CheckpointTs:449523817334964229 MinTableBarrierTs:449523817334964229 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523817334964229, checkpointTs: 449523817334964229, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523817334964229, checkpointTs: 449523817334964229, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8 {"id":"dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e17cb27c dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8 /tidb/cdc/default/default/changefeed/info/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {"upstream-id":7364998104416399251,"namespace":"default","changefeed-id":"7ce3deb2-1cd2-4297-878c-0ac02d25e66f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:27.374996846+08:00","start-ts":449523814831489026,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523814857703426} /tidb/cdc/default/default/changefeed/status/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {"checkpoint-ts":449523817334964229,"min-table-barrier-ts":449523817334964229,"admin-job-type":0} /tidb/cdc/default/default/task/position/dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998104416399251 {"id":7364998104416399251,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {UpstreamID:7364998104416399251 Namespace:default ID:7ce3deb2-1cd2-4297-878c-0ac02d25e66f SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:27.374996846 +0800 CST StartTs:449523814831489026 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003502000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523814857703426} {CheckpointTs:449523817334964229 MinTableBarrierTs:449523817334964229 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523817334964229, checkpointTs: 449523817334964229, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523817334964229, checkpointTs: 449523817334964229, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8 {"id":"dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797275} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e17cb27c dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8 /tidb/cdc/default/default/changefeed/info/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {"upstream-id":7364998104416399251,"namespace":"default","changefeed-id":"7ce3deb2-1cd2-4297-878c-0ac02d25e66f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:27.374996846+08:00","start-ts":449523814831489026,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523814857703426} /tidb/cdc/default/default/changefeed/status/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {"checkpoint-ts":449523817334964229,"min-table-barrier-ts":449523817334964229,"admin-job-type":0} /tidb/cdc/default/default/task/position/dce5dc61-8ceb-4fdc-95e3-e2179e8b9dc8/7ce3deb2-1cd2-4297-878c-0ac02d25e66f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998104416399251 {"id":7364998104416399251,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/04 12:34:37.892 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 20"] [2024/05/04 12:34:37.840 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table ddl_manager.finish_mark not exists for 67-th check, retry later check diff failed 4-th time, retry later [Sat May 4 12:34:38 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.79387940.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully test_kill_owner: pass wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_hang_up_owner [Sat May 4 12:34:40 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.30803082.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server1 --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8137.out cli tso query --pd=http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 12:34:39.851 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 68-th check, retry later check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:40 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03c6c1c3-8884-4465-9020-07497a5a4899 {"id":"03c6c1c3-8884-4465-9020-07497a5a4899","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797278} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b25ed0 03c6c1c3-8884-4465-9020-07497a5a4899 /tidb/cdc/default/default/upstream/7364998171798087430 {"id":7364998171798087430,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03c6c1c3-8884-4465-9020-07497a5a4899 {"id":"03c6c1c3-8884-4465-9020-07497a5a4899","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797278} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b25ed0 03c6c1c3-8884-4465-9020-07497a5a4899 /tidb/cdc/default/default/upstream/7364998171798087430 {"id":7364998171798087430,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03c6c1c3-8884-4465-9020-07497a5a4899 {"id":"03c6c1c3-8884-4465-9020-07497a5a4899","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797278} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b25ed0 03c6c1c3-8884-4465-9020-07497a5a4899 /tidb/cdc/default/default/upstream/7364998171798087430 {"id":7364998171798087430,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9046.out cli changefeed create --start-ts=449523817161949185 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 7fbe43e0-ebc9-4423-918b-e95665671e6b Info: {"upstream_id":7364998171798087430,"namespace":"default","id":"7fbe43e0-ebc9-4423-918b-e95665671e6b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:34:41.195453079+08:00","start_ts":449523817161949185,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523817161949185,"checkpoint_ts":449523817161949185,"checkpoint_time":"2024-05-04 12:34:36.161"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449523818201612289 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523818201612289 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "recordcount"="10" "requestdistribution"="uniform" "mysql.user"="root" "workload"="core" "operationcount"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "updateproportion"="0" "scanproportion"="0" "readallfields"="true" "readproportion"="0" "mysql.port"="4000" "threadcount"="2" "insertproportion"="0" "mysql.db"="multi_capture_1" ********************************************** Run finished, takes 8.829038ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2053.8, Avg(us): 1686, Min(us): 996, Max(us): 3924, 95th(us): 4000, 99th(us): 4000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:41 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8d5b675-5a93-4ad2-9574-accf779a251b {"id":"f8d5b675-5a93-4ad2-9574-accf779a251b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797279} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b998cf f8d5b675-5a93-4ad2-9574-accf779a251b /tidb/cdc/default/default/upstream/7364998164698770627 {"id":7364998164698770627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8d5b675-5a93-4ad2-9574-accf779a251b {"id":"f8d5b675-5a93-4ad2-9574-accf779a251b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797279} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b998cf f8d5b675-5a93-4ad2-9574-accf779a251b /tidb/cdc/default/default/upstream/7364998164698770627 {"id":7364998164698770627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8d5b675-5a93-4ad2-9574-accf779a251b {"id":"f8d5b675-5a93-4ad2-9574-accf779a251b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797279} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b998cf f8d5b675-5a93-4ad2-9574-accf779a251b /tidb/cdc/default/default/upstream/7364998164698770627 {"id":7364998164698770627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.7999.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-only-block-related-table ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.port"="4000" "operationcount"="0" "readallfields"="true" "scanproportion"="0" "threadcount"="2" "recordcount"="10" "mysql.db"="multi_capture_2" "mysql.user"="root" "dotransactions"="false" "readproportion"="0" "workload"="core" "insertproportion"="0" "updateproportion"="0" "requestdistribution"="uniform" ********************************************** Run finished, takes 9.855695ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1822.6, Avg(us): 1907, Min(us): 1176, Max(us): 4347, 95th(us): 5000, 99th(us): 5000 [2024/05/04 12:34:41.861 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] ***************** properties ***************** "mysql.port"="4000" "dotransactions"="false" "updateproportion"="0" "threadcount"="2" "workload"="core" "mysql.user"="root" "readproportion"="0" "scanproportion"="0" "mysql.db"="multi_capture_3" "mysql.host"="127.0.0.1" "operationcount"="0" "requestdistribution"="uniform" "insertproportion"="0" "readallfields"="true" "recordcount"="10" ********************************************** Run finished, takes 9.06249ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1999.9, Avg(us): 1744, Min(us): 1086, Max(us): 3965, 95th(us): 4000, 99th(us): 4000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "mysql.db"="testSync" "readallfields"="true" "scanproportion"="0" "requestdistribution"="uniform" "recordcount"="10" "insertproportion"="0" "updateproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "threadcount"="2" "readproportion"="0" "mysql.port"="4000" "mysql.user"="root" "dotransactions"="false" "workload"="core" ********************************************** Run finished, takes 5.727274ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2734.5, Avg(us): 1000, Min(us): 631, Max(us): 2411, 95th(us): 3000, 99th(us): 3000 Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7364998164698770627,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-04T12:34:42.248007212+08:00","start_ts":449523818735599617,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523818735599617,"checkpoint_ts":449523818735599617,"checkpoint_time":"2024-05-04 12:34:42.164"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "requestdistribution"="uniform" "readallfields"="true" "scanproportion"="0" "readproportion"="0" "threadcount"="2" "mysql.host"="127.0.0.1" "mysql.port"="4000" "dotransactions"="false" "recordcount"="10" "operationcount"="0" "insertproportion"="0" "updateproportion"="0" "workload"="core" "mysql.db"="multi_capture_4" "mysql.user"="root" ********************************************** Run finished, takes 9.310179ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1846.1, Avg(us): 1756, Min(us): 1073, Max(us): 3829, 95th(us): 4000, 99th(us): 4000 [Sat May 4 12:34:42 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.82618263.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 69-th check, retry later check diff failed 6-th time, retry later + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:34:42.896 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 28"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030eec60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523816726003720 MinTableBarrierTs:449523818823155719 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523816726003720, checkpointTs: 449523816726003720, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523816726003720, checkpointTs: 449523816726003720, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523816726003720, checkpointTs: 449523816726003720, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1d6fa49-9814-4ce6-9add-37c689c59492 {"id":"d1d6fa49-9814-4ce6-9add-37c689c59492","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cee8 d1d6fa49-9814-4ce6-9add-37c689c59492 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523816726003720,"min-table-barrier-ts":449523818823155719,"admin-job-type":0} /tidb/cdc/default/default/task/position/d1d6fa49-9814-4ce6-9add-37c689c59492/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030eec60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523816726003720 MinTableBarrierTs:449523818823155719 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523816726003720, checkpointTs: 449523816726003720, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523816726003720, checkpointTs: 449523816726003720, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523816726003720, checkpointTs: 449523816726003720, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1d6fa49-9814-4ce6-9add-37c689c59492 {"id":"d1d6fa49-9814-4ce6-9add-37c689c59492","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cee8 d1d6fa49-9814-4ce6-9add-37c689c59492 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523816726003720,"min-table-barrier-ts":449523818823155719,"admin-job-type":0} /tidb/cdc/default/default/task/position/d1d6fa49-9814-4ce6-9add-37c689c59492/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030eec60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523816726003720 MinTableBarrierTs:449523818823155719 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523816726003720, checkpointTs: 449523816726003720, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523816726003720, checkpointTs: 449523816726003720, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523816726003720, checkpointTs: 449523816726003720, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1d6fa49-9814-4ce6-9add-37c689c59492 {"id":"d1d6fa49-9814-4ce6-9add-37c689c59492","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cee8 d1d6fa49-9814-4ce6-9add-37c689c59492 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523816726003720,"min-table-barrier-ts":449523818823155719,"admin-job-type":0} /tidb/cdc/default/default/task/position/d1d6fa49-9814-4ce6-9add-37c689c59492/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + set +x [2024/05/04 12:34:43.873 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 70-th check, retry later check_redo_resolved_ts 7ce3deb2-1cd2-4297-878c-0ac02d25e66f 449523819065114628 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/7ce3deb2-1cd2-4297-878c-0ac02d25e66f/meta global resolved ts 449523818908090374 not forward to 449523819065114628 run task failed 1-th time, retry later table ddl_sequence.finish_mark not exists for 1-th check, retry later "is-owner": true, run task successfully check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_redo_resolved_ts 7ce3deb2-1cd2-4297-878c-0ac02d25e66f 449523819065114628 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/7ce3deb2-1cd2-4297-878c-0ac02d25e66f/meta run task successfully [2024/05/04 12:34:45.884 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:45 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0479f48-c23f-4994-ad2a-d5ea61764564 {"id":"e0479f48-c23f-4994-ad2a-d5ea61764564","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797282} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f3a e0479f48-c23f-4994-ad2a-d5ea61764564 /tidb/cdc/default/default/upstream/7364998178025653365 {"id":7364998178025653365,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0479f48-c23f-4994-ad2a-d5ea61764564 {"id":"e0479f48-c23f-4994-ad2a-d5ea61764564","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797282} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f3a e0479f48-c23f-4994-ad2a-d5ea61764564 /tidb/cdc/default/default/upstream/7364998178025653365 {"id":7364998178025653365,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0479f48-c23f-4994-ad2a-d5ea61764564 {"id":"e0479f48-c23f-4994-ad2a-d5ea61764564","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797282} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f3a e0479f48-c23f-4994-ad2a-d5ea61764564 /tidb/cdc/default/default/upstream/7364998178025653365 {"id":7364998178025653365,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 12:34:45 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.83118313.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07881d2c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf, pid:1300, start at 2024-05-04 12:34:43.187675878 +0800 CST m=+5.246778325 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:43.194 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:43.198 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:43.198 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07881d2c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf, pid:1300, start at 2024-05-04 12:34:43.187675878 +0800 CST m=+5.246778325 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:43.194 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:43.198 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:43.198 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07881dd40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf, pid:1385, start at 2024-05-04 12:34:43.234128452 +0800 CST m=+5.215814969 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:43.243 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:43.239 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:43.239 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } owner pid: 3085 owner id d1d6fa49-9814-4ce6-9add-37c689c59492 [Sat May 4 12:34:46 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.32163218.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... table ddl_sequence.finish_mark not exists for 2-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 71-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078829800019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0, pid:1370, start at 2024-05-04 12:34:43.978315016 +0800 CST m=+5.246440047 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:43.985 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:43.987 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:43.987 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078829800019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0, pid:1370, start at 2024-05-04 12:34:43.978315016 +0800 CST m=+5.246440047 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:43.985 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:43.987 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:43.987 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d07882a6c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-9570k-8qxk6-0bkl0, pid:1450, start at 2024-05-04 12:34:44.027083646 +0800 CST m=+5.241337769 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:36:44.033 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:34:43.995 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:24:43.995 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [2024/05/04 12:34:47.788 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/04 12:34:47.789 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/04 12:34:47.789 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/04 12:34:47.790 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/04 12:34:47.894 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [2024/05/04 12:34:47.793 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/04 12:34:47.793 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=449523817597370379] [resolvedTs=449523819445223425] [2024/05/04 12:34:47.794 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/04 12:34:47.794 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/04 12:34:47.794 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 12:34:47.794 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 12:34:47.794 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 12:34:47.797 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=3.936974ms] [2024/05/04 12:34:47.797 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=3.916567ms] [2024/05/04 12:34:47.798 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 12:34:47.800 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.801 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/04 12:34:47.802 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/04 12:34:47.802 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 12:34:47.802 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 12:34:47.802 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 12:34:47.805 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 12:34:47.807 +08:00] [INFO] [mysql_ddl_sink.go:107] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/04 12:34:47.807 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":449523817597370374,\"CommitTs\":449523817597370379,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_gbk\",\"Table\":\"GBKTABLE2\",\"TableID\":110,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2024/05/04 12:34:47.807 +08:00] [INFO] [mysql_ddl_sink.go:194] ["Start exec DDL"] [DDL="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [commitTs=449523817597370379] [namespace=] [changefeed=] [2024/05/04 12:34:47.809 +08:00] [INFO] [mysql_ddl_sink.go:127] ["Execute DDL failed, but error can be ignored"] [startTs=449523817597370374] [ddl="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [namespace=] [changefeed=] [error="Error 1050 (42S01): Table 'consistent_replicate_gbk.GBKTABLE2' already exists"] [2024/05/04 12:34:48.009 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449523819445223425] [2024/05/04 12:34:48.009 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449523819445223425] [2024/05/04 12:34:48.109 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=1] [appliedDDLCount=1] [currentCheckpoint=449523819445223425] [2024/05/04 12:34:48.109 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/04 12:34:48.109 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/04 12:34:48.109 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/04 12:34:48.109 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/04 12:34:48.109 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/04 12:34:48.109 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/04 12:34:48.109 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/04 12:34:48.110 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/04 12:34:48.109 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/04 12:34:48.110 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/04 12:34:48.110 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/04 12:34:48.109 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/04 12:34:48.109 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/04 12:34:48.110 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/04 12:34:48.110 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/04 12:34:48.110 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] Apply redo log successfully check diff successfully [Sat May 4 12:34:48 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT table ddl_sequence.finish_mark not exists for 3-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:48 GMT < Content-Length: 1259 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6367b011-10ff-4e30-b0b5-d2c44baea103 {"id":"6367b011-10ff-4e30-b0b5-d2c44baea103","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797285} /tidb/cdc/default/__cdc_meta__/capture/e0479f48-c23f-4994-ad2a-d5ea61764564 {"id":"e0479f48-c23f-4994-ad2a-d5ea61764564","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797282} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f3a e0479f48-c23f-4994-ad2a-d5ea61764564 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f6c 6367b011-10ff-4e30-b0b5-d2c44baea103 /tidb/cdc/default/default/upstream/7364998178025653365 {"id":7364998178025653365,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6367b011-10ff-4e30-b0b5-d2c44baea103 {"id":"6367b011-10ff-4e30-b0b5-d2c44baea103","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797285} /tidb/cdc/default/__cdc_meta__/capture/e0479f48-c23f-4994-ad2a-d5ea61764564 {"id":"e0479f48-c23f-4994-ad2a-d5ea61764564","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797282} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f3a e0479f48-c23f-4994-ad2a-d5ea61764564 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f6c 6367b011-10ff-4e30-b0b5-d2c44baea103 /tidb/cdc/default/default/upstream/7364998178025653365 {"id":7364998178025653365,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6367b011-10ff-4e30-b0b5-d2c44baea103 {"id":"6367b011-10ff-4e30-b0b5-d2c44baea103","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797285} /tidb/cdc/default/__cdc_meta__/capture/e0479f48-c23f-4994-ad2a-d5ea61764564 {"id":"e0479f48-c23f-4994-ad2a-d5ea61764564","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797282} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f3a e0479f48-c23f-4994-ad2a-d5ea61764564 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f6c 6367b011-10ff-4e30-b0b5-d2c44baea103 /tidb/cdc/default/default/upstream/7364998178025653365 {"id":7364998178025653365,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 12:34:48 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.83718373.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL [Sat May 4 12:34:48 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.26502652.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark exists check diff successfully + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.cli.2717.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 1-th time... check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026782d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523820396281868 MinTableBarrierTs:449523820396281868 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3029ece2-c57f-4846-980d-6e048841020c {"id":"3029ece2-c57f-4846-980d-6e048841020c","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797286} /tidb/cdc/default/__cdc_meta__/capture/d1d6fa49-9814-4ce6-9add-37c689c59492 {"id":"d1d6fa49-9814-4ce6-9add-37c689c59492","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cee8 d1d6fa49-9814-4ce6-9add-37c689c59492 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cf42 3029ece2-c57f-4846-980d-6e048841020c /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523820396281868,"min-table-barrier-ts":449523820396281868,"admin-job-type":0} /tidb/cdc/default/default/task/position/3029ece2-c57f-4846-980d-6e048841020c/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d1d6fa49-9814-4ce6-9add-37c689c59492/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026782d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523820396281868 MinTableBarrierTs:449523820396281868 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3029ece2-c57f-4846-980d-6e048841020c {"id":"3029ece2-c57f-4846-980d-6e048841020c","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797286} /tidb/cdc/default/__cdc_meta__/capture/d1d6fa49-9814-4ce6-9add-37c689c59492 {"id":"d1d6fa49-9814-4ce6-9add-37c689c59492","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cee8 d1d6fa49-9814-4ce6-9add-37c689c59492 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cf42 3029ece2-c57f-4846-980d-6e048841020c /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523820396281868,"min-table-barrier-ts":449523820396281868,"admin-job-type":0} /tidb/cdc/default/default/task/position/3029ece2-c57f-4846-980d-6e048841020c/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d1d6fa49-9814-4ce6-9add-37c689c59492/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026782d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523820396281868 MinTableBarrierTs:449523820396281868 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3029ece2-c57f-4846-980d-6e048841020c {"id":"3029ece2-c57f-4846-980d-6e048841020c","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797286} /tidb/cdc/default/__cdc_meta__/capture/d1d6fa49-9814-4ce6-9add-37c689c59492 {"id":"d1d6fa49-9814-4ce6-9add-37c689c59492","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cee8 d1d6fa49-9814-4ce6-9add-37c689c59492 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cf42 3029ece2-c57f-4846-980d-6e048841020c /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523820396281868,"min-table-barrier-ts":449523820396281868,"admin-job-type":0} /tidb/cdc/default/default/task/position/3029ece2-c57f-4846-980d-6e048841020c/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d1d6fa49-9814-4ce6-9add-37c689c59492/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "d1d6fa49-9814-4ce6-9add-37c689c59492" | grep id wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table ddl_only_block_related_table.finish_mark not exists for 4-th check, retry later [2024/05/04 12:34:49.906 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:34:50 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> "cluster-id": "default" "id": "3029ece2-c57f-4846-980d-6e048841020c", "cluster-id": "default" run task successfully + set +x + tso='449523820629065729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523820629065729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:34:50 CST 2024] <<<<<< START cdc server in tidb_mysql_test case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.27572759.out server --log-file /tmp/tidb_cdc_test/tidb_mysql_test/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tidb_mysql_test/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_sequence.finish_mark not exists for 4-th check, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:51 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/122926fe-267a-4169-a942-be9e2b9c99bd {"id":"122926fe-267a-4169-a942-be9e2b9c99bd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1d801d2 122926fe-267a-4169-a942-be9e2b9c99bd /tidb/cdc/default/default/upstream/7364998208553698920 {"id":7364998208553698920,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/122926fe-267a-4169-a942-be9e2b9c99bd {"id":"122926fe-267a-4169-a942-be9e2b9c99bd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1d801d2 122926fe-267a-4169-a942-be9e2b9c99bd /tidb/cdc/default/default/upstream/7364998208553698920 {"id":7364998208553698920,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/122926fe-267a-4169-a942-be9e2b9c99bd {"id":"122926fe-267a-4169-a942-be9e2b9c99bd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1d801d2 122926fe-267a-4169-a942-be9e2b9c99bd /tidb/cdc/default/default/upstream/7364998208553698920 {"id":7364998208553698920,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.2746.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Create changefeed successfully! ID: 976c1dff-c277-484c-9d8f-b90e9a453f85 Info: {"upstream_id":7364998208553698920,"namespace":"default","id":"976c1dff-c277-484c-9d8f-b90e9a453f85","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:34:51.96210436+08:00","start_ts":449523821274202114,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523821274202114,"checkpoint_ts":449523821274202114,"checkpoint_time":"2024-05-04 12:34:51.848"} PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:51 GMT < Content-Length: 1732 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6367b011-10ff-4e30-b0b5-d2c44baea103 {"id":"6367b011-10ff-4e30-b0b5-d2c44baea103","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797285} /tidb/cdc/default/__cdc_meta__/capture/97e423b6-0d5d-4361-bde7-cc2d2f64e03a {"id":"97e423b6-0d5d-4361-bde7-cc2d2f64e03a","address":"127.0.0.1:8303","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797288} /tidb/cdc/default/__cdc_meta__/capture/e0479f48-c23f-4994-ad2a-d5ea61764564 {"id":"e0479f48-c23f-4994-ad2a-d5ea61764564","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797282} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f3a e0479f48-c23f-4994-ad2a-d5ea61764564 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f6c 6367b011-10ff-4e30-b0b5-d2c44baea103 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f90 97e423b6-0d5d-4361-bde7-cc2d2f64e03a /tidb/cdc/default/default/upstream/7364998178025653365 {"id":7364998178025653365,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6367b011-10ff-4e30-b0b5-d2c44baea103 {"id":"6367b011-10ff-4e30-b0b5-d2c44baea103","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797285} /tidb/cdc/default/__cdc_meta__/capture/97e423b6-0d5d-4361-bde7-cc2d2f64e03a {"id":"97e423b6-0d5d-4361-bde7-cc2d2f64e03a","address":"127.0.0.1:8303","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797288} /tidb/cdc/default/__cdc_meta__/capture/e0479f48-c23f-4994-ad2a-d5ea61764564 {"id":"e0479f48-c23f-4994-ad2a-d5ea61764564","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797282} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f3a e0479f48-c23f-4994-ad2a-d5ea61764564 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f6c 6367b011-10ff-4e30-b0b5-d2c44baea103 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f90 97e423b6-0d5d-4361-bde7-cc2d2f64e03a /tidb/cdc/default/default/upstream/7364998178025653365 + grep -q 'failed to get info:' {"id":7364998178025653365,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6367b011-10ff-4e30-b0b5-d2c44baea103 {"id":"6367b011-10ff-4e30-b0b5-d2c44baea103","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797285} /tidb/cdc/default/__cdc_meta__/capture/97e423b6-0d5d-4361-bde7-cc2d2f64e03a {"id":"97e423b6-0d5d-4361-bde7-cc2d2f64e03a","address":"127.0.0.1:8303","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797288} /tidb/cdc/default/__cdc_meta__/capture/e0479f48-c23f-4994-ad2a-d5ea61764564 {"id":"e0479f48-c23f-4994-ad2a-d5ea61764564","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797282} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f3a e0479f48-c23f-4994-ad2a-d5ea61764564 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f6c 6367b011-10ff-4e30-b0b5-d2c44baea103 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1c39f90 97e423b6-0d5d-4361-bde7-cc2d2f64e03a /tidb/cdc/default/default/upstream/7364998178025653365 {"id":7364998178025653365,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8426.out cli changefeed create --start-ts=449523818201612289 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 capture_id: 3029ece2-c57f-4846-980d-6e048841020c ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '3029ece2-c57f-4846-980d-6e048841020c' 3029ece2-c57f-4846-980d-6e048841020c run task successfully test_hang_up_owner: pass coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_only_block_related_table.finish_mark exists [2024/05/04 12:34:51.918 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] Create changefeed successfully! ID: 9ca15bc2-35a1-43d5-9b72-5e7cd578493b Info: {"upstream_id":7364998178025653365,"namespace":"default","id":"9ca15bc2-35a1-43d5-9b72-5e7cd578493b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:34:52.118637488+08:00","start_ts":449523818201612289,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523818201612289,"checkpoint_ts":449523818201612289,"checkpoint_time":"2024-05-04 12:34:40.127"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... wait process 7943 exit for 1-th time... table ddl_sequence.finish_mark not exists for 5-th check, retry later check diff failed 3-th time, retry later wait process cdc.test exit for 2-th time... wait process 7943 exit for 2-th time... + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 24.69 secs (151057351 bytes/sec) [Pipeline] { + set +x table multi_capture_1.usertable not exists for 1-th check, retry later [Pipeline] cache wait process cdc.test exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (7943) - No such process wait process 7943 exit for 3-th time... process 7943 already exit [Sat May 4 12:34:53 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.80888090.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:53 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c413ddb-21fc-488b-955f-2ab6010a1e0a {"id":"3c413ddb-21fc-488b-955f-2ab6010a1e0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797291} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1e3e1d4 3c413ddb-21fc-488b-955f-2ab6010a1e0a /tidb/cdc/default/default/upstream/7364998212963583603 {"id":7364998212963583603,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c413ddb-21fc-488b-955f-2ab6010a1e0a {"id":"3c413ddb-21fc-488b-955f-2ab6010a1e0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797291} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1e3e1d4 3c413ddb-21fc-488b-955f-2ab6010a1e0a /tidb/cdc/default/default/upstream/7364998212963583603 {"id":7364998212963583603,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c413ddb-21fc-488b-955f-2ab6010a1e0a {"id":"3c413ddb-21fc-488b-955f-2ab6010a1e0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797291} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1e3e1d4 3c413ddb-21fc-488b-955f-2ab6010a1e0a /tidb/cdc/default/default/upstream/7364998212963583603 {"id":7364998212963583603,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 142 0 0 100 142 0 1519 --:--:-- --:--:-- --:--:-- 1526 mysql_test start go: downloading github.com/maxshuang/mysql-tester v0.0.2 [2024/05/04 12:34:53.933 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [2024/05/04 12:34:53.948 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark] [ts=449523821348913157] [2024/05/04 12:34:53.948 +08:00] [INFO] [case.go:498] ["all tables synced"] [endTs=449523821348913157] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_expire_owner [Sat May 4 12:34:54 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.33693371.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_owner.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' table ddl_sequence.finish_mark not exists for 6-th check, retry later 1 run task successfully table owner_resign.t1 exists check diff failed 1-th time, retry later check diff failed 4-th time, retry later table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later table ddl_sequence.finish_mark not exists for 7-th check, retry later go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20211122065349-4d16aabf900b go: downloading github.com/sirupsen/logrus v1.8.1 go: downloading github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3 go: downloading go.uber.org/atomic v1.9.0 go: downloading golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 go: downloading golang.org/x/text v0.3.7 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 go: downloading go.uber.org/zap v1.19.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 go: downloading go.uber.org/multierr v1.7.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364998164698770627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:42.248007212 +0800 CST StartTs:449523818735599617 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003993a70 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523818748706823} {CheckpointTs:449523822287912961 MinTableBarrierTs:449523822287912961 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4 {"id":"7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797294} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b999a3 7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364998164698770627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:34:42.248007212+08:00","start-ts":449523818735599617,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523818748706823} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449523822287912961,"min-table-barrier-ts":449523822549794818,"admin-job-type":0} /tidb/cdc/default/default/task/position/7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998164698770627 {"id":7364998164698770627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364998164698770627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:42.248007212 +0800 CST StartTs:449523818735599617 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003993a70 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523818748706823} {CheckpointTs:449523822287912961 MinTableBarrierTs:449523822287912961 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4 {"id":"7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797294} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b999a3 7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364998164698770627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:34:42.248007212+08:00","start-ts":449523818735599617,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523818748706823} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449523822287912961,"min-table-barrier-ts":449523822549794818,"admin-job-type":0} /tidb/cdc/default/default/task/position/7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998164698770627 {"id":7364998164698770627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364998164698770627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:42.248007212 +0800 CST StartTs:449523818735599617 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003993a70 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523818748706823} {CheckpointTs:449523822287912961 MinTableBarrierTs:449523822287912961 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449523822549794818, checkpointTs: 449523822287912961, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4 {"id":"7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797294} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b999a3 7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364998164698770627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:34:42.248007212+08:00","start-ts":449523818735599617,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523818748706823} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449523822287912961,"min-table-barrier-ts":449523822549794818,"admin-job-type":0} /tidb/cdc/default/default/task/position/7bf92371-c10f-4b5d-96fa-ecd2d59bd2e4/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998164698770627 {"id":7364998164698770627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff failed 1-th time, retry later check diff successfully [Sat May 4 12:34:56 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.29272929.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 5-th time, retry later [2024/05/04 12:34:56.934 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark1000] [2024/05/04 12:34:56.934 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark1000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003c78090 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523821182451726 MinTableBarrierTs:449523822506278915 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523821182451726, checkpointTs: 449523821182451726, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523821182451726, checkpointTs: 449523821182451726, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523821182451726, checkpointTs: 449523821182451726, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4aaee9c7-64ed-4f37-a732-a0f69c585448 {"id":"4aaee9c7-64ed-4f37-a732-a0f69c585448","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797294} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cfa0 4aaee9c7-64ed-4f37-a732-a0f69c585448 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523821182451726,"min-table-barrier-ts":449523822506278915,"admin-job-type":0} /tidb/cdc/default/default/task/position/4aaee9c7-64ed-4f37-a732-a0f69c585448/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003c78090 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523821182451726 MinTableBarrierTs:449523822506278915 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523821182451726, checkpointTs: 449523821182451726, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523821182451726, checkpointTs: 449523821182451726, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523821182451726, checkpointTs: 449523821182451726, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4aaee9c7-64ed-4f37-a732-a0f69c585448 {"id":"4aaee9c7-64ed-4f37-a732-a0f69c585448","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797294} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cfa0 4aaee9c7-64ed-4f37-a732-a0f69c585448 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523821182451726,"min-table-barrier-ts":449523822506278915,"admin-job-type":0} /tidb/cdc/default/default/task/position/4aaee9c7-64ed-4f37-a732-a0f69c585448/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003c78090 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523821182451726 MinTableBarrierTs:449523822506278915 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523821182451726, checkpointTs: 449523821182451726, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523821182451726, checkpointTs: 449523821182451726, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523821182451726, checkpointTs: 449523821182451726, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4aaee9c7-64ed-4f37-a732-a0f69c585448 {"id":"4aaee9c7-64ed-4f37-a732-a0f69c585448","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797294} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cfa0 4aaee9c7-64ed-4f37-a732-a0f69c585448 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523821182451726,"min-table-barrier-ts":449523822506278915,"admin-job-type":0} /tidb/cdc/default/default/task/position/4aaee9c7+ grep -q 'etcd info' -64ed-4f37-a732-a0f69c585448/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... "is-owner": true, run task successfully table ddl_sequence.finish_mark not exists for 8-th check, retry later check diff failed 6-th time, retry later [2024/05/04 12:34:58.934 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] [2024/05/04 12:34:59.029 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark1000] [ts=449523822934884357] [2024/05/04 12:34:59.029 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark1000] [2024/05/04 12:34:59.029 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523822934884357] [2024/05/04 12:34:59.032 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:34:59.035 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:34:59.121 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:34:59.127 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] check diff successfully check_ts_not_forward ddl-only-block-related-table [2024/05/04 12:34:59.830 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark2000] [2024/05/04 12:34:59.830 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark2000] table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:34:59 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/976c1dff-c277-484c-9d8f-b90e9a453f85 {UpstreamID:7364998208553698920 Namespace:default ID:976c1dff-c277-484c-9d8f-b90e9a453f85 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:51.96210436 +0800 CST StartTs:449523821274202114 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025f0bd0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523821300416515} {CheckpointTs:449523823161376773 MinTableBarrierTs:449523823161376773 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/122926fe-267a-4169-a942-be9e2b9c99bd {"id":"122926fe-267a-4169-a942-be9e2b9c99bd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797288} /tidb/cdc/default/__cdc_meta__/capture/50fb907c-0067-4205-a595-22f93e51ea3b {"id":"50fb907c-0067-4205-a595-22f93e51ea3b","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1d801d2 122926fe-267a-4169-a942-be9e2b9c99bd /tidb/cdc/default/__cdc_meta__/owner/22318f41e1d80266 50fb907c-0067-4205-a595-22f93e51ea3b /tidb/cdc/default/default/changefeed/info/976c1dff-c277-484c-9d8f-b90e9a453f85 {"upstream-id":7364998208553698920,"namespace":"default","changefeed-id":"976c1dff-c277-484c-9d8f-b90e9a453f85","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:51.96210436+08:00","start-ts":449523821274202114,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523821300416515} /tidb/cdc/default/default/changefeed/status/976c1dff-c277-484c-9d8f-b90e9a453f85 {"checkpoint-ts":449523823161376773,"min-table-barrier-ts":449523823161376773,"admin-job-type":0} /tidb/cdc/default/default/task/position/122926fe-267a-4169-a942-be9e2b9c99bd/976c1dff-c277-484c-9d8f-b90e9a453f85 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/50fb907c-0067-4205-a595-22f93e51ea3b/976c1dff-c277-484c-9d8f-b90e9a453f85 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998208553698920 {"id":7364998208553698920,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/976c1dff-c277-484c-9d8f-b90e9a453f85 {UpstreamID:7364998208553698920 Namespace:default ID:976c1dff-c277-484c-9d8f-b90e9a453f85 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:51.96210436 +0800 CST StartTs:449523821274202114 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025f0bd0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523821300416515} {CheckpointTs:449523823161376773 MinTableBarrierTs:449523823161376773 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/122926fe-267a-4169-a942-be9e2b9c99bd {"id":"122926fe-267a-4169-a942-be9e2b9c99bd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797288} /tidb/cdc/default/__cdc_meta__/capture/50fb907c-0067-4205-a595-22f93e51ea3b {"id":"50fb907c-0067-4205-a595-22f93e51ea3b","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1d801d2 122926fe-267a-4169-a942-be9e2b9c99bd /tidb/cdc/default/__cdc_meta__/owner/22318f41e1d80266 50fb907c-0067-4205-a595-22f93e51ea3b /tidb/cdc/default/default/changefeed/info/976c1dff-c277-484c-9d8f-b90e9a453f85 {"upstream-id":7364998208553698920,"namespace":"default","changefeed-id":"976c1dff-c277-484c-9d8f-b90e9a453f85","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:51.96210436+08:00","start-ts":449523821274202114,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523821300416515} /tidb/cdc/default/default/changefeed/status/976c1dff-c277-484c-9d8f-b90e9a453f85 {"checkpoint-ts":449523823161376773,"min-table-barrier-ts":449523823161376773,"admin-job-type":0} /tidb/cdc/default/default/task/position/122926fe-267a-4169-a942-be9e2b9c99bd/976c1dff-c277-484c-9d8f-b90e9a453f85 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /+ grep -q 'failed to get info:' tidb/cdc/default/default/task/position/50fb907c-0067-4205-a595-22f93e51ea3b/976c1dff-c277-484c-9d8f-b90e9a453f85 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998208553698920 {"id":7364998208553698920,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/976c1dff-c277-484c-9d8f-b90e9a453f85 {UpstreamID:7364998208553698920 Namespace:default ID:976c1dff-c277-484c-9d8f-b90e9a453f85 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:51.96210436 +0800 CST StartTs:449523821274202114 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025f0bd0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523821300416515} {CheckpointTs:449523823161376773 MinTableBarrierTs:449523823161376773 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/122926fe-267a-4169-a942-be9e2b9c99bd {"id":"122926fe-267a-4169-a942-be9e2b9c99bd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797288} /tidb/cdc/default/__cdc_meta__/capture/50fb907c-0067-4205-a595-22f93e51ea3b {"id":"50fb907c-0067-4205-a595-22f93e51ea3b","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1d801d2 122926fe-267a-4169-a942-be9e2b9c99bd /tidb/cdc/default/__cdc_meta__/owner/22318f41e1d80266 50fb907c-0067-4205-a595-22f93e51ea3b /tidb/cdc/default/default/changefeed/info/976c1dff-c277-484c-9d8f-b90e9a453f85 {"upstream-id":7364998208553698920,"namespace":"default","changefeed-id":"976c1dff-c277-484c-9d8f-b90e9a453f85","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:51.96210436+08:00","start-ts":449523821274202114,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523821300416515} /tidb/cdc/default/default/changefeed/status/976c1dff-c277-484c-9d8f-b90e9a453f85 {"checkpoint-ts":449523823161376773,"min-table-barrier-ts":449523823161376773,"admin-job-type":0} /tidb/cdc/default/default/task/position/122926fe-267a-4169-a942-be9e2b9c99bd/976c1dff-c277-484c-9d8f-b90e9a453f85 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/50fb907c-0067-4205-a595-22f93e51ea3b/976c1dff-c277-484c-9d8f-b90e9a453f85 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998208553698920 {"id":7364998208553698920,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 table ddl_sequence.finish_mark exists owner pid: 3374 owner id 4aaee9c7-64ed-4f37-a732-a0f69c585448 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '4aaee9c7-64ed-4f37-a732-a0f69c585448' /tidb/cdc/default/__cdc_meta__/owner/22318f41e163cfa0 run task successfully process status: S cdc.test cli capture list 2>&1 | grep '"is-owner": true' =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: mysql... <<================= check diff successfully check diff successfully wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table multi_capture_4.usertable exists check diff successfully ***************** properties ***************** "scanproportion"="0" "recordcount"="20" "updateproportion"="0" "mysql.user"="root" "dotransactions"="false" "requestdistribution"="uniform" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="multi_capture_1" "operationcount"="0" "readallfields"="true" "mysql.port"="4000" "threadcount"="2" "workload"="core" "insertproportion"="0" ********************************************** Run finished, takes 11.298015ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2069.2, Avg(us): 1076, Min(us): 489, Max(us): 1717, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "scanproportion"="0" "mysql.db"="multi_capture_2" "dotransactions"="false" "operationcount"="0" "mysql.user"="root" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "workload"="core" "readallfields"="true" "insertproportion"="0" "updateproportion"="0" "recordcount"="20" "requestdistribution"="uniform" "threadcount"="2" ********************************************** Run finished, takes 11.475701ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1953.5, Avg(us): 1087, Min(us): 510, Max(us): 1755, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "readproportion"="0" "recordcount"="20" "mysql.host"="127.0.0.1" "operationcount"="0" "mysql.db"="multi_capture_3" "updateproportion"="0" "insertproportion"="0" "mysql.user"="root" "readallfields"="true" "mysql.port"="4000" "dotransactions"="false" "scanproportion"="0" "threadcount"="2" "requestdistribution"="uniform" "workload"="core" ********************************************** Run finished, takes 12.035669ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1946.7, Avg(us): 1163, Min(us): 551, Max(us): 1757, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "dotransactions"="false" "threadcount"="2" "mysql.port"="4000" "updateproportion"="0" "operationcount"="0" "readallfields"="true" "scanproportion"="0" "mysql.user"="root" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "insertproportion"="0" "recordcount"="20" "readproportion"="0" "mysql.db"="multi_capture_4" "workload"="core" ********************************************** Run finished, takes 13.152625ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1676.0, Avg(us): 1148, Min(us): 663, Max(us): 2115, 95th(us): 2000, 99th(us): 3000 [2024/05/04 12:35:01.830 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later "is-owner": true, run task successfully test_expire_owner pass wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... [2024/05/04 12:35:02.228 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark3000] [2024/05/04 12:35:02.228 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark3000] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:35:02 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_owner_cleanup_stale_tasks [Sat May 4 12:35:03 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.35743576.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server1 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later [2024/05/04 12:35:03.844 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] The 1 times to try to start tidb cluster... [2024/05/04 12:35:03.930 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark2000] [ts=449523823931293701] [2024/05/04 12:35:03.930 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark2000] [2024/05/04 12:35:03.930 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523823931293701] [2024/05/04 12:35:03.932 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:03.939 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:04.019 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:04.032 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:04.228 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 3-th time... [2024/05/04 12:35:05.629 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark4000] [2024/05/04 12:35:05.629 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark4000] check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:35:06 CST 2024] <<<<<< run test case multi_capture success! >>>>>> [2024/05/04 12:35:06.244 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] [2024/05/04 12:35:06.419 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark3000] [ts=449523824831234053] [2024/05/04 12:35:06.419 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark3000] [2024/05/04 12:35:06.419 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523824831234053] [2024/05/04 12:35:06.421 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:06.425 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:06.438 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:06.520 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00127ca20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523824865574916 MinTableBarrierTs:449523824865574916 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523824865574916, checkpointTs: 449523824865574916, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523824865574916, checkpointTs: 449523824865574916, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523824865574916, checkpointTs: 449523824865574916, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 {"id":"3a717a87-a3eb-48fa-bf7e-35eaf6ebde34","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d004 3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523824865574916,"min-table-barrier-ts":449523824865574916,"admin-job-type":0} /tidb/cdc/default/default/task/position/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00127ca20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523824865574916 MinTableBarrierTs:449523824865574916 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523824865574916, checkpointTs: 449523824865574916, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523824865574916, checkpointTs: 449523824865574916, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523824865574916, checkpointTs: 449523824865574916, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 {"id":"3a717a87-a3eb-48fa-bf7e-35eaf6ebde34","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d004 3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523824865574916,"min-table-barrier-ts":449523824865574916,"admin-job-type":0} /tidb/cdc/default/default/task/position/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00127ca20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523824865574916 MinTableBarrierTs:449523824865574916 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523824865574916, checkpointTs: 449523824865574916, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523824865574916, checkpointTs: 449523824865574916, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523824865574916, checkpointTs: 449523824865574916, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 {"id":"3a717a87-a3eb-48fa-bf7e-35eaf6ebde34","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d004 3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523824865574916,"min-table-barrier-ts":449523824865574916,"admin-job-type":0} /tidb/cdc/default/default/task/position/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 2-th time, retry later [2024/05/04 12:35:07.629 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 "is-owner": true, run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... [2024/05/04 12:35:08.928 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark5000] [2024/05/04 12:35:08.928 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark5000] owner pid: 3579 owner id 3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 [Sat May 4 12:35:09 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.37103712.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later [2024/05/04 12:35:09.642 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] [2024/05/04 12:35:09.735 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark4000] [ts=449523825722523663] [2024/05/04 12:35:09.735 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark4000] [2024/05/04 12:35:09.735 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523825722523663] [2024/05/04 12:35:09.819 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:09.824 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:09.828 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:09.833 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/04 12:35:10.928 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later [2024/05/04 12:35:12.230 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark6000] [2024/05/04 12:35:12.230 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark6000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00158f7a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523826438438918 MinTableBarrierTs:449523826438438918 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27745403-24e6-426d-9d43-47e03371a133 {"id":"27745403-24e6-426d-9d43-47e03371a133","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797309} /tidb/cdc/default/__cdc_meta__/capture/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 {"id":"3a717a87-a3eb-48fa-bf7e-35eaf6ebde34","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d004 3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d054 27745403-24e6-426d-9d43-47e03371a133 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523826438438918,"min-table-barrier-ts":449523826438438918,"admin-job-type":0} /tidb/cdc/default/default/task/position/27745403-24e6-426d-9d43-47e03371a133/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00158f7a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523826438438918 MinTableBarrierTs:449523826438438918 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27745403-24e6-426d-9d43-47e03371a133 {"id":"27745403-24e6-426d-9d43-47e03371a133","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797309} /tidb/cdc/default/__cdc_meta__/capture/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 {"id":"3a717a87-a3eb-48fa-bf7e-35eaf6ebde34","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d004 3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d054 27745403-24e6-426d-9d43-47e03371a133 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523826438438918,"min-table-barrier-ts":449523826438438918,"admin-job-type":0} /tidb/cdc/default/default/task/position/27745403-24e6-426d-9d43-47e03371a133/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/task/position/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00158f7a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523826438438918 MinTableBarrierTs:449523826438438918 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27745403-24e6-426d-9d43-47e03371a133 {"id":"27745403-24e6-426d-9d43-47e03371a133","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797309} /tidb/cdc/default/__cdc_meta__/capture/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 {"id":"3a717a87-a3eb-48fa-bf7e-35eaf6ebde34","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d004 3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d054 27745403-24e6-426d-9d43-47e03371a133 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523826438438918,"min-table-barrier-ts":449523826438438918,"admin-job-type":0} /tidb/cdc/default/default/task/position/27745403-24e6-426d-9d43-47e03371a133/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "3a717a87-a3eb-48fa-bf7e-35eaf6ebde34" | grep id [2024/05/04 12:35:12.940 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] [2024/05/04 12:35:13.036 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark5000] [ts=449523826526257157] [2024/05/04 12:35:13.036 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark5000] [2024/05/04 12:35:13.036 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523826526257157] [2024/05/04 12:35:13.119 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:13.122 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:13.127 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:13.131 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later "cluster-id": "default" "id": "27745403-24e6-426d-9d43-47e03371a133", "cluster-id": "default" run task successfully [2024/05/04 12:35:14.230 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 18.73 secs (199111754 bytes/sec) [Pipeline] { [Pipeline] cache capture_id: 27745403-24e6-426d-9d43-47e03371a133 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0789e5100005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:6903, start at 2024-05-04 12:35:12.328222494 +0800 CST m=+5.365225069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:12.335 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:12.324 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:12.324 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0789e5100005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:6903, start at 2024-05-04 12:35:12.328222494 +0800 CST m=+5.365225069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:12.335 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:12.324 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:12.324 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0789e75c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:6947, start at 2024-05-04 12:35:12.472066499 +0800 CST m=+5.455739642 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:12.482 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:12.471 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:12.471 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 12:35:15.930 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark7000] [2024/05/04 12:35:15.930 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark7000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... [2024/05/04 12:35:16.243 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] [2024/05/04 12:35:16.333 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark6000] [ts=449523827312689157] [2024/05/04 12:35:16.333 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark6000] [2024/05/04 12:35:16.333 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523827312689157] [2024/05/04 12:35:16.420 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:16.423 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:16.428 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:16.433 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 12:35:17 CST 2024] <<<<<< START cdc server in consistent_replicate_nfs case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_nfs.82288230.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 12:35:17.931 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] check diff successfully check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 0 [Sat May 4 12:35:18 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.38533855.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... [2024/05/04 12:35:18.724 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark8000] [2024/05/04 12:35:18.724 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark8000] wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078a46e40001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq, pid:6836, start at 2024-05-04 12:35:18.585835698 +0800 CST m=+5.421255100 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:18.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:18.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:18.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:35:19 CST 2024] <<<<<< run test case owner_resign success! >>>>>> [2024/05/04 12:35:19.944 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] [2024/05/04 12:35:20.042 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark7000] [ts=449523828212891653] [2024/05/04 12:35:20.042 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark7000] [2024/05/04 12:35:20.042 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523828212891653] [2024/05/04 12:35:20.120 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:20.125 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:20.130 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:20.134 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:20 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11001339-e3a8-4069-bcff-a4dd22be531b {"id":"11001339-e3a8-4069-bcff-a4dd22be531b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e24e78cd 11001339-e3a8-4069-bcff-a4dd22be531b /tidb/cdc/default/default/upstream/7364998337393307728 {"id":7364998337393307728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11001339-e3a8-4069-bcff-a4dd22be531b {"id":"11001339-e3a8-4069-bcff-a4dd22be531b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e24e78cd 11001339-e3a8-4069-bcff-a4dd22be531b /tidb/cdc/default/default/upstream/7364998337393307728 {"id":7364998337393307728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11001339-e3a8-4069-bcff-a4dd22be531b {"id":"11001339-e3a8-4069-bcff-a4dd22be531b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e24e78cd 11001339-e3a8-4069-bcff-a4dd22be531b /tidb/cdc/default/default/upstream/7364998337393307728 {"id":7364998337393307728,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:20.725 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078a46e40001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq, pid:6836, start at 2024-05-04 12:35:18.585835698 +0800 CST m=+5.421255100 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:18.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:18.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:18.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078a49480003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-br2gk-nk79c-c2qgq, pid:6919, start at 2024-05-04 12:35:18.744365963 +0800 CST m=+5.526167394 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:18.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:18.738 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:18.738 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/04 12:35:13.572 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/05/04 12:35:13.572 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/05/04 12:35:14.093 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLseea522a6_a6b0_4c51_a5e3_5bfa5bfd2fa4"] [2024/05/04 12:35:14.100 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsabbb13d3_c48f_47d4_a503_80692df26294"] [2024/05/04 12:35:14.101 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd20b6f9c_7094_48f8_a9ac_9a2ea87735c8"] [2024/05/04 12:35:14.102 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs17d1ec66_65b3_49db_8cf3_d65d83775277"] [2024/05/04 12:35:14.103 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa93473c6_0d98_4abd_a40f_7ca30facdcb4"] [2024/05/04 12:35:14.104 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa0f9cfb7_79ca_4de0_a60f_5f3705be9410"] [2024/05/04 12:35:14.105 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsbd633131_4037_4d4c_9806_3f42b4367c27"] [2024/05/04 12:35:14.108 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf39595ec_c735_476b_b935_81f1305a65f3"] [2024/05/04 12:35:14.391 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/04 12:35:14.395 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/04 12:35:14.589 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/04 12:35:14.589 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/04 12:35:15.077 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:15.082 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:15.084 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:15.087 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:15.089 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:15.090 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:15.167 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:15.167 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:15.284 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:15.294 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:15.368 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:15.368 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:15.370 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:15.372 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:15.375 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:15.382 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:15.471 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/04 12:35:15.483 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/04 12:35:15.986 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/04 12:35:15.992 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/04 12:35:16.066 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/04 12:35:16.072 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/04 12:35:16.167 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:16.171 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:16.171 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:16.175 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:16.194 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:16.196 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:16.265 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:16.269 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:16.374 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:16.470 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:16.477 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:16.478 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:16.478 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:16.480 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:16.481 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:16.485 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:16.575 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/04 12:35:16.579 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/04 12:35:17.273 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:17.276 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:17.281 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:17.290 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:17.297 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:17.297 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:17.370 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:17.374 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:17.384 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:17.467 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/04 12:35:17.469 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/04 12:35:17.471 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:17.484 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:17.487 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:17.488 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:17.489 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:17.492 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:17.494 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:17.582 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/04 12:35:17.584 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/04 12:35:18.368 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:18.368 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:18.373 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:18.382 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:18.392 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:18.393 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:18.393 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:18.471 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:18.484 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:18.493 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:18.579 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:18.583 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:18.585 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:18.593 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:18.595 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:18.665 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:18.682 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/04 12:35:18.685 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/04 12:35:18.885 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/04 12:35:18.886 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/04 12:35:18.894 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/04 12:35:18.894 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/04 12:35:19.480 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:19.483 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:19.488 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:19.566 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:19.571 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:19.575 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:19.580 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:19.581 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:19.587 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:19.596 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:19.683 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:19.684 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:19.768 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:19.774 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:19.777 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:19.778 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:19.791 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/04 12:35:19.791 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/04 12:35:20.469 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/04 12:35:20.476 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/04 12:35:20.589 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:20.672 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:20.673 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:20.682 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:20.685 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:20.691 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:20.767 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:20.770 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:20.773 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:20.774 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:20.795 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:20.868 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:20.886 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:20.889 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:20.967 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:20.970 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:20.979 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/04 12:35:21.067 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ***************** properties ***************** "insertproportion"="0" "mysql.db"="consistent_replicate_nfs" "mysql.port"="4000" "updateproportion"="0" "scanproportion"="0" "workload"="core" "fieldcount"="3" "readproportion"="0" "readallfields"="true" "operationcount"="0" "mysql.user"="root" "mysql.host"="127.0.0.1" "threadcount"="10" "recordcount"="50000" "dotransactions"="false" "requestdistribution"="uniform" ********************************************** [2024/05/04 12:35:21.673 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:21.680 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:21.684 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:21.693 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:21.765 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:21.774 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:21.782 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:21.783 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:21.787 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/04 12:35:21.790 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/04 12:35:21.791 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003447ef0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523827224870919 MinTableBarrierTs:449523827224870919 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27745403-24e6-426d-9d43-47e03371a133 {"id":"27745403-24e6-426d-9d43-47e03371a133","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797309} /tidb/cdc/default/__cdc_meta__/capture/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 {"id":"3a717a87-a3eb-48fa-bf7e-35eaf6ebde34","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797303} /tidb/cdc/default/__cdc_meta__/capture/cd85390f-8931-424b-9349-baeb98c0b187 {"id":"cd85390f-8931-424b-9349-baeb98c0b187","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797318} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d004 3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d054 27745403-24e6-426d-9d43-47e03371a133 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d0c0 cd85390f-8931-424b-9349-baeb98c0b187 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523827224870919,"min-table-barrier-ts":449523827224870919,"admin-job-type":0} /tidb/cdc/default/default/task/position/27745403-24e6-426d-9d43-47e03371a133/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cd85390f-8931-424b-9349-baeb98c0b187/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003447ef0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523827224870919 MinTableBarrierTs:449523827224870919 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27745403-24e6-426d-9d43-47e03371a133 {"id":"27745403-24e6-426d-9d43-47e03371a133","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797309} /tidb/cdc/default/__cdc_meta__/capture/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 {"id":"3a717a87-a3eb-48fa-bf7e-35eaf6ebde34","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797303} /tidb/cdc/default/__cdc_meta__/capture/cd85390f-8931-424b-9349-baeb98c0b187 {"id":"cd85390f-8931-424b-9349-baeb98c0b187","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797318} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d004 3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d054 27745403-24e6-426d-9d43-47e03371a133 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d0c0 cd85390f-8931-424b-9349-baeb98c0b187 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523827224870919,"min-table-barrier-ts":449523827224870919,"admin-job-type":0} /tidb/cdc/default/default/task/position/27745403-24e6-426d-9d43-47e03371a133/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cd85390f-8931-424b-9349-baeb98c0b187/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003447ef0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523827224870919 MinTableBarrierTs:449523827224870919 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27745403-24e6-426d-9d43-47e03371a133 {"id":"27745403-24e6-426d-9d43-47e03371a133","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797309} /tidb/cdc/default/__cdc_meta__/capture/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 {"id":"3a717a87-a3eb-48fa-bf7e-35eaf6ebde34","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797303} /tidb/cdc/default/__cdc_meta__/capture/cd85390f-8931-424b-9349-baeb98c0b187 {"id":"cd85390f-8931-424b-9349-baeb98c0b187","address":"127.0.0.1:8302","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797318} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d004 3a717a87-a3eb-48fa-bf7e-35eaf6ebde34 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d054 27745403-24e6-426d-9d43-47e03371a133 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d0c0 cd85390f-8931-424b-9349-baeb98c0b187 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","+ grep -q 'etcd info' error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523827224870919,"min-table-barrier-ts":449523827224870919,"admin-job-type":0} /tidb/cdc/default/default/task/position/27745403-24e6-426d-9d43-47e03371a133/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3a717a87-a3eb-48fa-bf7e-35eaf6ebde34/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cd85390f-8931-424b-9349-baeb98c0b187/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' run task failed 1-th time, retry later [2024/05/04 12:35:21.868 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/04 12:35:21.882 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:21.888 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:21.998 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:22.001 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:22.071 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:22.086 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:22.087 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:22.030 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark9000] [2024/05/04 12:35:22.030 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark9000] check diff failed 1-th time, retry later [2024/05/04 12:35:22.172 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/04 12:35:22.172 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:22.186 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:22.791 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:22.866 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:22.870 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:22.884 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:22.894 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:22.895 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:22.817 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2024/05/04 12:35:22.921 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark8000] [ts=449523829042839557] [2024/05/04 12:35:22.921 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark8000] [2024/05/04 12:35:22.921 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523829042839557] [2024/05/04 12:35:22.923 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:22.926 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:22.931 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:22.936 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:22.977 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:22.981 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:23.071 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:23.082 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] build mysql_test successfully run all mysql test cases time="2024-05-04T12:35:20+08:00" level=info msg="running tests: [alter_table bigint composite_index concurrent_ddl date_formats drop mysql_replace]" time="2024-05-04T12:35:20+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000126000 {{} {} 0} {0 0} [0xc000140000] map[] 0 1 0xc00012a000 false map[0xc000140000:map[0xc000140000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [2024/05/04 12:35:23.183 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:23.188 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:23.266 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:23.283 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:23.284 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:23.373 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:23.374 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/04 12:35:23.376 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/04 12:35:23.376 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/04 12:35:23.382 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/04 12:35:23.878 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:23.883 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:23.888 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:23.976 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:23.979 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:23.981 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:23.988 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8282.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 2-th time, retry later [2024/05/04 12:35:24.072 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:24.170 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:24.191 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' run task failed 2-th time, retry later [2024/05/04 12:35:24.031 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] [2024/05/04 12:35:24.283 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:24.287 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:24.373 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:24.388 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:24.391 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:24.481 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/04 12:35:24.483 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:24.484 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:24.790 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/04 12:35:24.868 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/04 12:35:24.872 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/04 12:35:24.879 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/04 12:35:24.989 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:24.992 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:25.070 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:25.088 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:25.088 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:25.093 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:25.165 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:25.174 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] + set +x + tso='449523829659664386 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523829659664386 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:35:25 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.83218323.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 12:35:25.366 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:25.382 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:25.473 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:25.484 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:25.488 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:25.573 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:25.579 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:25.822 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark10000] [2024/05/04 12:35:25.822 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark10000] [2024/05/04 12:35:25.592 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:25.669 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/04 12:35:25.670 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] check diff failed 3-th time, retry later [2024/05/04 12:35:26.044 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] [2024/05/04 12:35:26.077 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:26.082 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:26.085 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:26.133 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark9000] [ts=449523829890875400] [2024/05/04 12:35:26.134 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark9000] [2024/05/04 12:35:26.134 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523829890875400] [2024/05/04 12:35:26.136 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:26.138 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:26.142 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:26.146 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:26.183 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:26.185 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:26.193 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:26.193 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:26.196 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/04 12:35:26.266 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:26.278 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/04 12:35:26.575 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:26.586 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:26.681 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:26.691 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:26.774 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:26.779 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:26.786 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078ab3c40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:10101, start at 2024-05-04 12:35:25.587626712 +0800 CST m=+5.145717872 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:25.594 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:25.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:25.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078ab3c40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:10101, start at 2024-05-04 12:35:25.587626712 +0800 CST m=+5.145717872 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:25.594 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:25.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:25.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078ab6100009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-hmd9p-tvz3q-2t5mg, pid:10186, start at 2024-05-04 12:35:25.707975764 +0800 CST m=+5.207998435 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:25.714 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:25.700 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:25.700 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/04 12:35:26.878 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:26.887 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/04 12:35:26.887 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/04 12:35:27.077 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:27.082 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:27.168 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:27.276 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:27.277 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:27.373 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:27.384 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:27.392 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:27.767 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:27.769 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:27.785 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/04 12:35:27.865 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:27.866 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/04 12:35:27.874 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:27.878 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/04 12:35:27.878 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:27.884 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/04 12:35:27.885 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:27.891 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:27.980 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/04 12:35:27.983 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:27.984 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/04 12:35:27.822 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/04 12:35:28.372 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' check diff failed 4-th time, retry later [2024/05/04 12:35:28.481 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:35:28.482 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:35:28.579 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:35:28.581 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:28 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3283e79c-9ff3-4088-bcb9-c34d8a26bb18 {"id":"3283e79c-9ff3-4088-bcb9-c34d8a26bb18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797325} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e26498d7 3283e79c-9ff3-4088-bcb9-c34d8a26bb18 /tidb/cdc/default/default/upstream/7364998358290329512 {"id":7364998358290329512,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3283e79c-9ff3-4088-bcb9-c34d8a26bb18 {"id":"3283e79c-9ff3-4088-bcb9-c34d8a26bb18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797325} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e26498d7 3283e79c-9ff3-4088-bcb9-c34d8a26bb18 /tidb/cdc/default/default/upstream/7364998358290329512 {"id":7364998358290329512,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3283e79c-9ff3-4088-bcb9-c34d8a26bb18 {"id":"3283e79c-9ff3-4088-bcb9-c34d8a26bb18","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797325} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e26498d7 3283e79c-9ff3-4088-bcb9-c34d8a26bb18 /tidb/cdc/default/default/upstream/7364998358290329512 {"id":7364998358290329512,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8385.out cli changefeed create --start-ts=449523829659664386 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ [2024/05/04 12:35:28.528 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark11000] [2024/05/04 12:35:28.528 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark11000] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11490.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/04 12:35:28.678 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:35:28.889 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:35:28.973 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:35:28.983 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:35:29.067 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:35:29.072 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:35:29.079 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/04 12:35:29.083 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:35:29.088 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] Create changefeed successfully! ID: a4537dae-dd5a-4901-a884-aa5f6d574fcc Info: {"upstream_id":7364998358290329512,"namespace":"default","id":"a4537dae-dd5a-4901-a884-aa5f6d574fcc","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:35:28.996212226+08:00","start_ts":449523829659664386,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523829659664386,"checkpoint_ts":449523829659664386,"checkpoint_time":"2024-05-04 12:35:23.836"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/04 12:35:29.268 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/04 12:35:29.281 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:35:29.283 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/04 12:35:29.384 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:35:29.385 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:29.575 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] "is-owner": true, run task successfully nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/04 12:35:29.780 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/04 12:35:29.784 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/04 12:35:29.870 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/04 12:35:29.885 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/04 12:35:29.890 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/04 12:35:24.497 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 2 another is 6"] [2024/05/04 12:35:29.835 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/04 12:35:29.931 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark10000] [ts=449523830834331654] [2024/05/04 12:35:29.931 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark10000] [2024/05/04 12:35:29.931 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523830834331654] [2024/05/04 12:35:29.933 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:29.936 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:29.941 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:29.946 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:30.091 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/04 12:35:30.096 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/04 12:35:30.272 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/04 12:35:30.277 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/04 12:35:30.284 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/04 12:35:30.288 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/04 12:35:30.367 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] check diff failed 5-th time, retry later + set +x + tso='449523831027531777 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523831027531777 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:35:30 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1152811530.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 [2024/05/04 12:35:30.528 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/04 12:35:30.483 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/04 12:35:30.508 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/04 12:35:30.714 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/protobuf v1.33.0 [2024/05/04 12:35:30.783 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] [2024/05/04 12:35:30.826 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/04 12:35:30.841 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/05/04 12:35:30.880 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc0c55766_9334_490b_8401_e81621c34b48"] [2024/05/04 12:35:30.884 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa719f864_9ca1_4feb_9147_aca0ca811c4b"] [2024/05/04 12:35:30.887 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb3c72565_bc60_4fc5_b873_66c2492e3fde"] [2024/05/04 12:35:30.890 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs78c8ab43_2ee3_4dd1_a8cc_446a857513c5"] go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda [2024/05/04 12:35:31.185 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading golang.org/x/text v0.14.0 [2024/05/04 12:35:31.371 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd76cd1e6_938c_48c8_989a_3c1abe790d76"] [2024/05/04 12:35:31.384 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb20698c6_88dd_42c5_b48b_d60fe6121fed"] [2024/05/04 12:35:31.465 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs468aaa04_4a51_4b64_9d78_51e65cc961c8"] INSERT - Takes(s): 10.0, Count: 19891, OPS: 1991.9, Avg(us): 5005, Min(us): 757, Max(us): 98805, 95th(us): 31000, 99th(us): 84000 [2024/05/04 12:35:31.586 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] [2024/05/04 12:35:31.592 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:31.671 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:31.673 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:31.680 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:31.680 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:31.687 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:31.688 +08:00] [INFO] [main.go:178] ["0 insert success: 1400"] [2024/05/04 12:35:31.691 +08:00] [INFO] [main.go:199] ["0 delete success: 700"] nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/04 12:35:31.769 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:31.772 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:32.029 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark12000] [2024/05/04 12:35:32.029 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark12000] [2024/05/04 12:35:32.088 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:32.175 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:32.185 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:32.191 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:32.198 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] check diff failed 6-th time, retry later [2024/05/04 12:35:32.468 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:32.482 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:32.617 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/04 12:35:32.568 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] [2024/05/04 12:35:32.676 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:32.677 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:32.720 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark11000] [ts=449523831646978053] [2024/05/04 12:35:32.720 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark11000] [2024/05/04 12:35:32.720 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523831646978053] [2024/05/04 12:35:32.724 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:32.727 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:32.731 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:32.825 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 15.23 secs (244988273 bytes/sec) [Pipeline] { [Pipeline] sh [2024/05/04 12:35:32.966 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:33.067 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:33.074 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] sh go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/coocood/freecache v1.2.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading go.uber.org/mock v0.4.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading cloud.google.com/go v0.112.2 [2024/05/04 12:35:33.278 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:33.372 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=1dd5f9ec-de1e-42e6-9701-bde0de3f6669 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-rgqf5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-rgqf5 pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.12 [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078b0df8001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:9662, start at 2024-05-04 12:35:31.357066366 +0800 CST m=+6.081434989 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:31.365 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:31.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:31.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078b0df8001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:9662, start at 2024-05-04 12:35:31.357066366 +0800 CST m=+6.081434989 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:31.365 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:31.326 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:31.326 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078b0f640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-0q39v-0x5bp-jqkgb, pid:9726, start at 2024-05-04 12:35:31.449987556 +0800 CST m=+6.115014146 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:31.460 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:31.467 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:31.467 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:33 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0655eb05-fb00-4e23-9790-33a708901fc3 {"id":"0655eb05-fb00-4e23-9790-33a708901fc3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797330} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e28513ce 0655eb05-fb00-4e23-9790-33a708901fc3 /tidb/cdc/default/default/upstream/7364998396311473018 {"id":7364998396311473018,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0655eb05-fb00-4e23-9790-33a708901fc3 {"id":"0655eb05-fb00-4e23-9790-33a708901fc3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797330} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e28513ce 0655eb05-fb00-4e23-9790-33a708901fc3 /tidb/cdc/default/default/upstream/7364998396311473018 {"id":7364998396311473018,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0655eb05-fb00-4e23-9790-33a708901fc3 {"id":"0655eb05-fb00-4e23-9790-33a708901fc3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797330} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e28513ce 0655eb05-fb00-4e23-9790-33a708901fc3 /tidb/cdc/default/default/upstream/7364998396311473018 {"id":7364998396311473018,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11583.out cli changefeed create --start-ts=449523831027531777 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=457cf3c8-10cf-4fe9-867e-6337b5bcea34 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-t604k GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-t604k pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/04 12:35:33.968 +08:00] [INFO] [main.go:178] ["0 insert success: 1500"] [Pipeline] // stage [Pipeline] } go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 Create changefeed successfully! ID: 5710de1b-432c-42bb-bd50-b13813c38eb7 Info: {"upstream_id":7364998396311473018,"namespace":"default","id":"5710de1b-432c-42bb-bd50-b13813c38eb7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:35:34.142695386+08:00","start_ts":449523831027531777,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523831027531777,"checkpoint_ts":449523831027531777,"checkpoint_time":"2024-05-04 12:35:29.054"} PASS [2024/05/04 12:35:34.074 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:34.178 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G17 [2024/05/04 12:35:34.029 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [Pipeline] { go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/04 12:35:34.281 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:34.465 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:34.472 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=4e413ea7-f06e-4614-a4a1-82befcf773ad BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-wtlx8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap_tiflow_pull_cdc_integration_test_1731-wtlx8 pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... [2024/05/04 12:35:34.667 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:34.684 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:34.685 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] check diff successfully go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 [2024/05/04 12:35:34.770 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:34.868 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] [2024/05/04 12:35:34.975 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsea9d1df5_c003_4661_b1e1_5dda8391b82b"] [2024/05/04 12:35:35.069 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:35.075 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:35.079 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:35.165 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:35.281 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.10972.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/04 12:35:35.679 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:35.769 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] + set +x Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/04 12:35:35.782 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:35.890 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:35.969 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:35.971 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:35.978 +08:00] [INFO] [main.go:178] ["0 insert success: 1600"] [2024/05/04 12:35:35.987 +08:00] [INFO] [main.go:199] ["0 delete success: 800"] table resourcecontrol.finish_mark not exists for 1-th check, retry later empty select id, val from test.availability1 where id=1 run task successfully test_owner_cleanup_stale_tasks pass [2024/05/04 12:35:35.832 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark13000] [2024/05/04 12:35:35.832 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark13000] [2024/05/04 12:35:36.044 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/04 12:35:36.133 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark12000] [ts=449523832450711557] [2024/05/04 12:35:36.134 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark12000] [2024/05/04 12:35:36.134 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523832450711557] [2024/05/04 12:35:36.136 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:36.139 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:36.218 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:36.228 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:36.071 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:36.074 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:36.274 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:36.279 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:36.281 +08:00] [INFO] [main.go:178] ["1 insert success: 2100"] wait process cdc.test exit for 1-th time... [2024/05/04 12:35:36.372 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:36.377 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:36.387 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:36.473 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:36.474 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:36.666 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... [2024/05/04 12:35:37.173 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:37.191 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:37.276 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_owner_retryable_error [Sat May 4 12:35:37 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/capture/capture-campaign-compacted-error=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.41134115.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449523832855199745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523832855199745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449523832855199745 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sat May 4 12:35:37 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1100211004.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 12:35:37.380 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:37.383 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:37.386 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:37.465 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:37.565 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:37.586 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] Starting Upstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... [2024/05/04 12:35:37.673 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:37.767 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:37.779 +08:00] [INFO] [main.go:178] ["1 insert success: 2200"] [2024/05/04 12:35:37.882 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 12:35:37.969 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:37.974 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:37.981 +08:00] [INFO] [main.go:178] ["0 insert success: 1700"] [2024/05/04 12:35:37.982 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:38.066 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:37.832 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 [2024/05/04 12:35:38.366 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:38.389 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] go: downloading github.com/jmespath/go-jmespath v0.4.0 table resourcecontrol.finish_mark exists check diff successfully ***************** properties ***************** "readallfields"="true" "updateproportion"="0" "insertproportion"="0" "mysql.user"="root" "dotransactions"="false" "scanproportion"="0" "threadcount"="2" "mysql.host"="127.0.0.1" "mysql.db"="testSync" "readproportion"="0" "workload"="core" "operationcount"="0" "mysql.port"="4000" "requestdistribution"="uniform" "recordcount"="10" ********************************************** Run finished, takes 5.176238ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3051.8, Avg(us): 894, Min(us): 536, Max(us): 1996, 95th(us): 2000, 99th(us): 2000 [2024/05/04 12:35:38.472 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:38.579 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:38.583 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:38.530 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark14000] [2024/05/04 12:35:38.530 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark14000] wait process cdc.test exit for 1-th time... [2024/05/04 12:35:38.679 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:38.683 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:38.695 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:38.769 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:38.778 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:38.872 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] check diff failed 1-th time, retry later [2024/05/04 12:35:39.067 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:39.076 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:39.080 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:39.169 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:39.173 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] wait process cdc.test exit for 2-th time... go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd [2024/05/04 12:35:39.480 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:39.577 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:39.670 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:35:39 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> [2024/05/04 12:35:39.784 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:39.785 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:39.891 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:39.972 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:39.983 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:39.990 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:39.844 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2024/05/04 12:35:39.929 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark13000] [ts=449523833403342853] [2024/05/04 12:35:39.929 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark13000] [2024/05/04 12:35:39.929 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523833403342853] [2024/05/04 12:35:39.931 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:39.934 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:39.948 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:40.020 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later [2024/05/04 12:35:40.073 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:40.083 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] check diff failed 2-th time, retry later [2024/05/04 12:35:40.272 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:40.283 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:40.289 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:40.293 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:40.376 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:40.388 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:40.481 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:40.530 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/04 12:35:40.574 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00430cfc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523832467750919 MinTableBarrierTs:449523833778470920 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523834040614920, checkpointTs: 449523832467750919, state: Prepared span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523834040614920, checkpointTs: 449523832467750919, state: Prepared span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523834040614920, checkpointTs: 449523832467750919, state: Prepared *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 {"id":"0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d173 0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523832467750919,"min-table-barrier-ts":449523834040614920,"admin-job-type":0} /tidb/cdc/default/default/task/position/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00430cfc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523832467750919 MinTableBarrierTs:449523833778470920 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523834040614920, checkpointTs: 449523832467750919, state: Prepared span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523834040614920, checkpointTs: 449523832467750919, state: Prepared span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523834040614920, checkpointTs: 449523832467750919, state: Prepared *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 {"id":"0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d173 0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523832467750919,"min-table-barrier-ts":449523834040614920,"admin-job-type":0} /tidb/cdc/default/default/task/position/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00430cfc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523832467750919 MinTableBarrierTs:449523833778470920 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523834040614920, checkpointTs: 449523832467750919, state: Prepared span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523834040614920, checkpointTs: 449523832467750919, state: Prepared span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523834040614920, checkpointTs: 449523832467750919, state: Prepared *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 {"id":"0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d173 0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523832467750919,"min-table-barrier-ts":449523834040614920,"admin-job-type":0} /tidb/cdc/default/default/task/position/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:40 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f100f529-5127-4377-920e-29a04e548485 {"id":"f100f529-5127-4377-920e-29a04e548485","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e29405d4 f100f529-5127-4377-920e-29a04e548485 /tidb/cdc/default/default/upstream/7364998418433676074 {"id":7364998418433676074,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f100f529-5127-4377-920e-29a04e548485 {"id":"f100f529-5127-4377-920e-29a04e548485","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e29405d4 f100f529-5127-4377-920e-29a04e548485 /tidb/cdc/default/default/upstream/7364998418433676074 {"id":7364998418433676074,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f100f529-5127-4377-920e-29a04e548485 {"id":"f100f529-5127-4377-920e-29a04e548485","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e29405d4 f100f529-5127-4377-920e-29a04e548485 /tidb/cdc/default/default/upstream/7364998418433676074 {"id":7364998418433676074,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449523832855199745 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11060.out cli changefeed create --start-ts=449523832855199745 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml [2024/05/04 12:35:40.881 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:40.968 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Create changefeed successfully! ID: test-1 Info: {"upstream_id":7364998418433676074,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T12:35:41.077405642+08:00","start_ts":449523832855199745,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523832855199745,"checkpoint_ts":449523832855199745,"checkpoint_time":"2024-05-04 12:35:36.026"} PASS [2024/05/04 12:35:41.175 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:41.188 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:41.192 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:41.269 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release coverage: 2.5% of statements in github.com/pingcap/tiflow/... ./t/alter_table.test: ok! 64 test cases passed, take time 17.966633912 s time="2024-05-04T12:35:38+08:00" level=info msg="run test [alter_table] ok" time="2024-05-04T12:35:38+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0003101d0 {{} {} 0} {0 0} [0xc00038c000] map[] 0 1 0xc000300780 false map[0xc00038c000:map[0xc00038c000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [2024/05/04 12:35:41.283 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:41.369 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:41.484 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:41.489 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table test.finish_mark not exists for 3-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc already exists) [2024/05/04 12:35:41.571 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:41.579 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:41.579 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:41.588 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:41.590 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:41.669 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:41.772 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:41.774 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] "is-owner": true, run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/04 12:35:42.065 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:42.067 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 28.495251496s"] [2024/05/04 12:35:42.278 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:42.280 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... [2024/05/04 12:35:42.230 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark15000] [2024/05/04 12:35:42.230 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark15000] [2024/05/04 12:35:42.369 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:42.472 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:42.508 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:42.593 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed [2024/05/04 12:35:42.546 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/04 12:35:42.721 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark14000] [ts=449523834215989254] [2024/05/04 12:35:42.721 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark14000] [2024/05/04 12:35:42.721 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523834215989254] [2024/05/04 12:35:42.724 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:42.728 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:42.733 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:42.677 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:42.678 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:42.766 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:42.772 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:42.784 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:42.872 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:42.872 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:42.872 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] check diff failed 3-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2314 0 --:--:-- --:--:-- --:--:-- 2326 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-04 12:35:36.026","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-04 12:35:42.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '12:35:36.026","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '12:35:42.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '12:35:36.026","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '12:35:42.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-04 12:35:36.026' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '12:35:36.026","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '12:35:42.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '12:35:36.026","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '12:35:42.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-04 12:35:42' + echo 'sink_checkpoint_ts is 2024-05-04' 12:35:36.026 sink_checkpoint_ts is 2024-05-04 12:35:36.026 ++ date -d '2024-05-04 12:35:36.026' +%s + checkpoint_timestamp=1714797336 ++ date -d '2024-05-04 12:35:42' +%s + current_timestamp=1714797342 + '[' 6 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' [2024/05/04 12:35:42.818 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:42.881 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:43.065 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore [2024/05/04 12:35:43.268 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table test.finish_mark not exists for 4-th check, retry later Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:43.573 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:43.575 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] INSERT - Takes(s): 20.0, Count: 30659, OPS: 1534.0, Avg(us): 6501, Min(us): 757, Max(us): 102457, 95th(us): 74000, 99th(us): 85000 [2024/05/04 12:35:43.667 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:43.770 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:43.784 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:43.869 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:43.881 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] owner pid: 4118 owner id 0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 [Sat May 4 12:35:43 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/owner-run-with-error=1*return(true);github.com/pingcap/tiflow/cdc/capture/capture-resign-failed=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.42444246.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 12:35:43.982 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:43.983 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:43.995 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:44.086 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:44.093 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078bc49c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf, pid:4228, start at 2024-05-04 12:35:43.05138642 +0800 CST m=+5.206516610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:43.057 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:43.015 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:43.015 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078bc49c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf, pid:4228, start at 2024-05-04 12:35:43.05138642 +0800 CST m=+5.206516610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:43.057 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:43.015 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:43.015 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078bc55c0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-j6n6k-019xs-d2zwf, pid:4308, start at 2024-05-04 12:35:43.080912296 +0800 CST m=+5.166879000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:43.087 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:43.063 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:43.063 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/04 12:35:44.173 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:44.271 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:44.279 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:44.370 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:44.230 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:44.578 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:44.581 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:44.669 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow [Pipeline] { check diff failed 4-th time, retry later [2024/05/04 12:35:44.682 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:44.780 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [Pipeline] cache [2024/05/04 12:35:44.975 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:45.069 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:45.173 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:45.177 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:45.127 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark16000] [2024/05/04 12:35:45.127 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark16000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:45.279 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:45.293 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:45.371 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:45.371 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:45.480 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:45.681 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:45.682 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:45.682 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:45.687 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table test.finish_mark not exists for 5-th check, retry later [2024/05/04 12:35:45.767 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:45.868 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:45.877 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table test.t1 exists + sleep 5 [Sat May 4 12:35:46 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.55485550.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 12:35:46.171 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:35:46.177 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:35:46.280 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:46.383 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:35:46.386 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:46.317 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/04 12:35:46.338 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark15000] [ts=449523835177009157] [2024/05/04 12:35:46.338 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark15000] [2024/05/04 12:35:46.338 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523835177009157] [2024/05/04 12:35:46.419 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:46.422 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:46.426 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:46.430 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:46.589 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:46.687 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:35:46.691 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:35:46.703 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:46.704 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:46.707 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:46.711 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] check diff failed 5-th time, retry later [2024/05/04 12:35:46.778 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:47.169 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:47.127 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003894fc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523835364442114 MinTableBarrierTs:449523835626586115 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 {"id":"0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797337} /tidb/cdc/default/__cdc_meta__/capture/87ccef48-5bc9-4620-8488-a7a02f928385 {"id":"87ccef48-5bc9-4620-8488-a7a02f928385","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797343} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d173 0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d1c9 87ccef48-5bc9-4620-8488-a7a02f928385 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523835364442114,"min-table-barrier-ts":449523835626586115,"admin-job-type":0} /tidb/cdc/default/default/task/position/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/87ccef48-5bc9-4620-8488-a7a02f928385/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003894fc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523835364442114 MinTableBarrierTs:449523835626586115 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 {"id":"0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797337} /tidb/cdc/default/__cdc_meta__/capture/87ccef48-5bc9-4620-8488-a7a02f928385 {"id":"87ccef48-5bc9-4620-8488-a7a02f928385","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797343} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d173 0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d1c9 87ccef48-5bc9-4620-8488-a7a02f928385 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523835364442114,"min-table-barrier-ts":449523835626586115,"admin-job-type":0} /tidb/cdc/default/default/task/position/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/87ccef48-5bc9-4620-8488-a7a02f928385/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003894fc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523835364442114 MinTableBarrierTs:449523835626586115 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 {"id":"0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797337} /tidb/cdc/default/__cdc_meta__/capture/87ccef48-5bc9-4620-8488-a7a02f928385 {"id":"87ccef48-5bc9-4620-8488-a7a02f928385","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797343} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d173 0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d1c9 87ccef48-5bc9-4620-8488-a7a02f928385 /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523835364442114,"min-table-barrier-ts":449523835626586115,"admin-job-type":0} /tidb/cdc/default/default/task/position/0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/87ccef48-5bc9-4620-8488-a7a02f928385/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728" | grep id [2024/05/04 12:35:47.369 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:35:47.374 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:35:47.468 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:35:47.468 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:35:47.475 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:35:47.478 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:47.570 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] table test.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:47.794 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:47.874 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs93f8c7d9_7188_4006_a683_d59ddf28b8f5"] [2024/05/04 12:35:47.876 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsdb0eff3b_4ad7_482c_9364_b372a1848a6b"] [2024/05/04 12:35:47.897 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/04 12:35:47.912 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:48.008 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "cluster-id": "default" "id": "87ccef48-5bc9-4620-8488-a7a02f928385", "cluster-id": "default" run task successfully [2024/05/04 12:35:48.079 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7afd7c8d_ba5e_4b60_9785_1316ec1663c7"] [2024/05/04 12:35:48.476 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs03f3a0a6_aa8a_4270_b49b_7516f07d7de8"] [2024/05/04 12:35:48.431 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark17000] [2024/05/04 12:35:48.431 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark17000] [2024/05/04 12:35:48.575 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:35:48.682 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:48.682 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:48.690 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:48.693 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc2404d84_0112_44f1_b736_7dd242766976"] [2024/05/04 12:35:48.766 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:48.795 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/04 12:35:48.870 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:49.217 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2024/05/04 12:35:49.077 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:49.079 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:49.265 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3e7f2bc0_0e51_4543_8b6a_f7a349fd376c"] [2024/05/04 12:35:49.278 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] check diff failed 6-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:49 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/746785ed-52da-4b58-b762-61351f365ff0 {"id":"746785ed-52da-4b58-b762-61351f365ff0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797346} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2c8e6cb 746785ed-52da-4b58-b762-61351f365ff0 /tidb/cdc/default/default/upstream/7364998467286568367 {"id":7364998467286568367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/746785ed-52da-4b58-b762-61351f365ff0 {"id":"746785ed-52da-4b58-b762-61351f365ff0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797346} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2c8e6cb 746785ed-52da-4b58-b762-61351f365ff0 /tidb/cdc/default/default/upstream/7364998467286568367 {"id":7364998467286568367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/746785ed-52da-4b58-b762-61351f365ff0 {"id":"746785ed-52da-4b58-b762-61351f365ff0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797346} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2c8e6cb 746785ed-52da-4b58-b762-61351f365ff0 /tidb/cdc/default/default/upstream/7364998467286568367 {"id":7364998467286568367,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/04 12:35:49.233 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark16000] [ts=449523836028977157] [2024/05/04 12:35:49.233 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark16000] [2024/05/04 12:35:49.233 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523836028977157] [2024/05/04 12:35:49.236 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:49.239 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:49.323 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:49.328 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:49.389 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:49.485 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:49.486 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] table test.finish_mark not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:49.582 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:49.679 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:49.771 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:49.775 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] \033[0;36m<<< Run all test success >>>\033[0m ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:35:50.071 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] capture_id: 87ccef48-5bc9-4620-8488-a7a02f928385 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 35 100 35 0 0 26697 0 --:--:-- --:--:-- --:--:-- 35000 { "status": true, "message": "" }ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728' 0cb0adc7-c6e7-4ce8-b30b-bb7c859e9728 run task successfully ps -C cdc.test -o pid= | awk '{print $1}' | wc -l | grep 2 2 run task successfully test_owner_retryable_error pass [2024/05/04 12:35:50.081 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:50.182 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:50.190 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:50.195 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:50.295 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:50.303 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] wait process cdc.test exit for 1-th time... [2024/05/04 12:35:50.370 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:50.381 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:50.472 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:50.490 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:50.491 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:50.772 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:50.789 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:50.431 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078c1078000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx, pid:1234, start at 2024-05-04 12:35:47.880320391 +0800 CST m=+5.371953542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:47.887 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:47.870 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:47.870 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078c1078000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx, pid:1234, start at 2024-05-04 12:35:47.880320391 +0800 CST m=+5.371953542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:47.887 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:47.870 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:47.870 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078c11300003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-rgqf5-t2969-hqtfx, pid:1320, start at 2024-05-04 12:35:47.917630415 +0800 CST m=+5.344813295 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:47.924 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:47.916 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:47.916 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/04 12:35:50.967 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:50.984 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:50.986 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_gap_between_watch_capture [Sat May 4 12:35:51 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.44114413.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' check diff successfully ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2832 0 --:--:-- --:--:-- --:--:-- 2858 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-04 12:35:48.327","puller_resolved_ts":"2024-05-04 12:35:42.926","last_synced_ts":"2024-05-04 12:35:43.427","now_ts":"2024-05-04 12:35:50.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-04' '12:35:48.327","puller_resolved_ts":"2024-05-04' '12:35:42.926","last_synced_ts":"2024-05-04' '12:35:43.427","now_ts":"2024-05-04' '12:35:50.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-04' '12:35:48.327","puller_resolved_ts":"2024-05-04' '12:35:42.926","last_synced_ts":"2024-05-04' '12:35:43.427","now_ts":"2024-05-04' '12:35:50.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 [2024/05/04 12:35:51.188 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:51.271 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:51.272 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:51.284 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 8-th check, retry later [2024/05/04 12:35:51.382 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:51.396 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:51.399 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:51.486 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs715da958_e39c_4a59_87b3_70afad89a6ca"] [2024/05/04 12:35:51.575 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078c29c80011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r, pid:1156, start at 2024-05-04 12:35:49.51585005 +0800 CST m=+6.324267463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:49.523 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:49.490 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:49.490 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078c29c80011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r, pid:1156, start at 2024-05-04 12:35:49.51585005 +0800 CST m=+6.324267463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:49.523 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:49.490 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:49.490 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078c29d40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r, pid:1213, start at 2024-05-04 12:35:49.515351896 +0800 CST m=+6.259612413 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:49.523 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:49.493 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:49.493 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/04 12:35:51.729 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark18000] [2024/05/04 12:35:51.729 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark18000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) INSERT - Takes(s): 30.0, Count: 42521, OPS: 1418.0, Avg(us): 7033, Min(us): 757, Max(us): 102457, 95th(us): 76000, 99th(us): 85000 [2024/05/04 12:35:51.668 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:51.687 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:51.772 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:51.787 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:51.787 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsbb7ee7de_e184_47f0_b524_985fe69b23b6"] [2024/05/04 12:35:42.845 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/04 12:35:42.846 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7364997812849846074] [2024/05/04 12:35:42.846 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/04 12:35:42.846 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/04 12:35:42.846 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/04 12:35:42.846 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/04 12:35:42.847 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/04 12:35:42.847 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7364997812849846074] [2024/05/04 12:35:42.848 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/04 12:35:42.848 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/04 12:35:42.848 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/04 12:35:42.848 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/04 12:35:42.849 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/05/04 12:35:42.849 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/04 12:35:42.851 +08:00] [INFO] [store_cache.go:477] ["change store resolve state"] [store=2] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] [2024/05/04 12:35:51.982 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:52.068 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:52.081 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:52.086 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:52.379 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:52.482 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:52.569 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:52.580 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:52.677 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:52.678 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:52.444 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] [2024/05/04 12:35:52.620 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark17000] [ts=449523836933373957] [2024/05/04 12:35:52.621 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark17000] [2024/05/04 12:35:52.621 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523836933373957] [2024/05/04 12:35:52.624 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:52.628 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:52.864 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/04 12:35:52.864 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/05/04 12:35:52.864 +08:00] [INFO] [tso_dispatcher.go:214] ["exit tso requests cancel loop"] [2024/05/04 12:35:52.864 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/05/04 12:35:52.864 +08:00] [INFO] [tso_client.go:140] ["closing tso client"] [2024/05/04 12:35:52.864 +08:00] [INFO] [tso_dispatcher.go:268] ["exit tso dispatcher loop"] [2024/05/04 12:35:52.864 +08:00] [INFO] [tso_dispatcher.go:455] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/04 12:35:52.864 +08:00] [INFO] [tso_dispatcher.go:380] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/05/04 12:35:52.864 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/04 12:35:52.864 +08:00] [INFO] [tso_client.go:145] ["close tso client"] [2024/05/04 12:35:52.864 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/04 12:35:52.864 +08:00] [INFO] [tso_client.go:155] ["tso client is closed"] [2024/05/04 12:35:52.864 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] [2024/05/04 12:35:52.776 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:52.783 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:52.798 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:52.878 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:35:52.881 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:52.882 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:35:52.719 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:52.725 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Aborted by Jenkins Admin [2024/05/04 12:35:53.068 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:53.071 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:53.166 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:53.173 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [Sat May 4 12:35:52 CST 2024] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.26752677.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 9-th check, retry later ./t/bigint.test: ok! 138 test cases passed, take time 13.90939965 s time="2024-05-04T12:35:53+08:00" level=info msg="run test [bigint] ok" time="2024-05-04T12:35:53+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000310110 {{} {} 0} {0 0} [0xc00038c5a0] map[] 0 1 0xc0003003c0 false map[0xc00038c5a0:map[0xc00038c5a0:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078c5be00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc, pid:1178, start at 2024-05-04 12:35:52.718690662 +0800 CST m=+9.275473375 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:52.729 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:52.696 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:52.696 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078c5be00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc, pid:1178, start at 2024-05-04 12:35:52.718690662 +0800 CST m=+9.275473375 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:52.729 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:52.696 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:52.696 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078c5c940008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-wtlx8-mmqds-hvcdc, pid:1251, start at 2024-05-04 12:35:52.753503241 +0800 CST m=+9.259553023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:37:52.863 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:35:53.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:25:53.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/04 12:35:53.573 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:53.670 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:35:53.679 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:53.729 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] [2024/05/04 12:35:53.774 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:53.974 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:53.976 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2564.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/04 12:35:54.183 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:54.267 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:54.273 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:54.282 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/clustered_index/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/clustered_index/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/04 12:35:54.367 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:35:54.369 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:54.382 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:54.384 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:54.467 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:54.475 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/05/04 12:35:54.871 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:54.882 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:54.981 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:55.081 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:55.129 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark19000] [2024/05/04 12:35:55.129 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark19000] [2024/05/04 12:35:55.177 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:55.185 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] ./t/composite_index.test: ok! 12 test cases passed, take time 1.549782622 s time="2024-05-04T12:35:55+08:00" level=info msg="run test [composite_index] ok" time="2024-05-04T12:35:55+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000310188 {{} {} 0} {0 0} [0xc000140090] map[] 0 1 0xc000300600 false map[0xc000140090:map[0xc000140090:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" + set +x + tso='449523837538402305 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523837538402305 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:35:55 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.26062608.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 12:35:55.383 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:55.386 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:55.472 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:55.475 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:35:55.483 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:35:55.490 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:55.571 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:55.572 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:55.579 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:55.580 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:55.817 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] [2024/05/04 12:35:55.776 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:55.776 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:55.868 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] table test.finish_mark not exists for 10-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/05/04 12:35:55.923 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark18000] [ts=449523837859790853] [2024/05/04 12:35:55.923 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark18000] [2024/05/04 12:35:55.923 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523837859790853] [2024/05/04 12:35:55.926 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:56.019 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:56.025 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:56.030 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:55.966 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:56 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e1f01789-c05b-4d6b-baba-f5260dd4fe12 {"id":"e1f01789-c05b-4d6b-baba-f5260dd4fe12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797353} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2d953cb e1f01789-c05b-4d6b-baba-f5260dd4fe12 /tidb/cdc/default/default/upstream/7364998494435701589 {"id":7364998494435701589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e1f01789-c05b-4d6b-baba-f5260dd4fe12 {"id":"e1f01789-c05b-4d6b-baba-f5260dd4fe12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797353} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2d953cb e1f01789-c05b-4d6b-baba-f5260dd4fe12 /tidb/cdc/default/default/upstream/7364998494435701589 {"id":7364998494435701589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e1f01789-c05b-4d6b-baba-f5260dd4fe12 {"id":"e1f01789-c05b-4d6b-baba-f5260dd4fe12","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797353} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2d953cb e1f01789-c05b-4d6b-baba-f5260dd4fe12 /tidb/cdc/default/default/upstream/7364998494435701589 {"id":7364998494435701589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: ce7558bc-b15b-45af-a5b8-dbe795f7b6cc Info: {"upstream_id":7364998494435701589,"namespace":"default","id":"ce7558bc-b15b-45af-a5b8-dbe795f7b6cc","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-04T12:35:56.223474139+08:00","start_ts":449523837283860481,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523837283860481,"checkpoint_ts":449523837283860481,"checkpoint_time":"2024-05-04 12:35:52.920"} [2024/05/04 12:35:56.172 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:56.178 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:56.377 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] check diff failed 1-th time, retry later [2024/05/04 12:35:56.465 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:56.474 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:35:56.478 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:56.574 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:35:56.584 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:56.591 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] ./t/concurrent_ddl.test: ok! 8 test cases passed, take time 1.02488907 s time="2024-05-04T12:35:56+08:00" level=info msg="run test [concurrent_ddl] ok" time="2024-05-04T12:35:56+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0003a6060 {{} {} 0} {0 0} [0xc0003e2000] map[] 0 1 0xc00039e180 false map[0xc0003e2000:map[0xc0003e2000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [Sat May 4 12:35:56 CST 2024] <<<<<< START cdc server in clustered_index case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.clustered_index.26492651.out server --log-file /tmp/tidb_cdc_test/clustered_index/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/clustered_index/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 12:35:56.666 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:56.673 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:56.681 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:56.776 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:56.791 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:56.870 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:56.877 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:56.985 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:57.067 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:57.366 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:35:57.129 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] [2024/05/04 12:35:57.471 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:35:57.476 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:57.576 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table test.finish_mark not exists for 11-th check, retry later [2024/05/04 12:35:57.768 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:57.774 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:57.778 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:35:57.785 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:57.865 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:35:57.897 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025b3170 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523836150874116 MinTableBarrierTs:449523836413018116 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43ede0f7-2ae0-474f-9692-1802786dfacc {"id":"43ede0f7-2ae0-474f-9692-1802786dfacc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797351} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d21f 43ede0f7-2ae0-474f-9692-1802786dfacc /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523836150874116,"min-table-barrier-ts":449523836413018116,"admin-job-type":0} /tidb/cdc/default/default/task/position/43ede0f7-2ae0-474f-9692-1802786dfacc/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025b3170 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523836150874116 MinTableBarrierTs:449523836413018116 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43ede0f7-2ae0-474f-9692-1802786dfacc {"id":"43ede0f7-2ae0-474f-9692-1802786dfacc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797351} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d21f 43ede0f7-2ae0-474f-9692-1802786dfacc /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523836150874116,"min-table-barrier-ts":449523836413018116,"admin-job-type":0} /tidb/cdc/default/default/task/position/43ede0f7-2ae0-474f-9692-1802786dfacc/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025b3170 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523836150874116 MinTableBarrierTs:449523836413018116 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43ede0f7-2ae0-474f-9692-1802786dfacc {"id":"43ede0f7-2ae0-474f-9692-1802786dfacc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797351} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d21f 43ede0f7-2ae0-474f-9692-1802786dfacc /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523836150874116,"min-table-barrier-ts":449523836413018116,"admin-job-type":0} /tidb/cdc/default/default/task/position/43ede0f7-2ae0-474f-9692-1802786dfacc/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2024/05/04 12:35:57.967 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:58.071 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:58.083 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:58.166 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:58.166 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:58.180 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:58.331 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20001] [2024/05/04 12:35:58.331 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/04 12:35:58.332 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20002] [2024/05/04 12:35:58.332 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/04 12:35:58.334 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20003] [2024/05/04 12:35:58.334 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/04 12:35:58.335 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20004] [2024/05/04 12:35:58.335 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] check diff failed 2-th time, retry later Run finished, takes 35.544909308s INSERT - Takes(s): 35.5, Count: 50000, OPS: 1407.2, Avg(us): 7080, Min(us): 757, Max(us): 102457, 95th(us): 76000, 99th(us): 85000 table consistent_replicate_nfs.usertable exists wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... [2024/05/04 12:35:58.283 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:35:58.384 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:35:58.407 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark20000] [2024/05/04 12:35:58.407 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20004] [2024/05/04 12:35:58.466 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:58.478 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:58 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e068230-cbb3-40ab-a1bc-71fc782e26d0 {"id":"4e068230-cbb3-40ab-a1bc-71fc782e26d0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797355} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2dbedd6 4e068230-cbb3-40ab-a1bc-71fc782e26d0 /tidb/cdc/default/default/upstream/7364998496871340478 {"id":7364998496871340478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e068230-cbb3-40ab-a1bc-71fc782e26d0 {"id":"4e068230-cbb3-40ab-a1bc-71fc782e26d0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797355} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2dbedd6 4e068230-cbb3-40ab-a1bc-71fc782e26d0 /tidb/cdc/default/default/upstream/7364998496871340478 {"id":7364998496871340478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e068230-cbb3-40ab-a1bc-71fc782e26d0 {"id":"4e068230-cbb3-40ab-a1bc-71fc782e26d0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797355} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2dbedd6 4e068230-cbb3-40ab-a1bc-71fc782e26d0 /tidb/cdc/default/default/upstream/7364998496871340478 {"id":7364998496871340478,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 8cf8e15d-b996-4df9-9a80-a1b536f3db66 Info: {"upstream_id":7364998496871340478,"namespace":"default","id":"8cf8e15d-b996-4df9-9a80-a1b536f3db66","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:35:58.773546128+08:00","start_ts":449523837538402305,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523837538402305,"checkpoint_ts":449523837538402305,"checkpoint_time":"2024-05-04 12:35:53.891"} wait process cdc.test exit for 3-th time... [2024/05/04 12:35:58.779 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:58.786 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:58.872 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:35:58.880 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:58.971 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:35:58.983 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:58.990 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:59.142 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] [2024/05/04 12:35:59.156 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark19000] [ts=449523838698651654] [2024/05/04 12:35:59.156 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark19000] [2024/05/04 12:35:59.156 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523838698651654] [2024/05/04 12:35:59.159 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:59.162 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:59.172 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:59.181 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:35:59.184 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:59.194 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] "is-owner": true, run task successfully table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table test.t2 not exists for 1-th check, retry later [2024/05/04 12:35:59.220 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:35:59.224 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:35:59.224 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark20000] [2024/05/04 12:35:59.280 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:59.295 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:35:59.386 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:35:59.386 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:59.395 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:35:59 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bbd90462-05cb-4d04-aab9-c7ae44936d82 {"id":"bbd90462-05cb-4d04-aab9-c7ae44936d82","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797356} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2dcf0d9 bbd90462-05cb-4d04-aab9-c7ae44936d82 /tidb/cdc/default/default/upstream/7364998495670293450 {"id":7364998495670293450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bbd90462-05cb-4d04-aab9-c7ae44936d82 {"id":"bbd90462-05cb-4d04-aab9-c7ae44936d82","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797356} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2dcf0d9 bbd90462-05cb-4d04-aab9-c7ae44936d82 /tidb/cdc/default/default/upstream/7364998495670293450 {"id":7364998495670293450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bbd90462-05cb-4d04-aab9-c7ae44936d82 {"id":"bbd90462-05cb-4d04-aab9-c7ae44936d82","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797356} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e2dcf0d9 bbd90462-05cb-4d04-aab9-c7ae44936d82 /tidb/cdc/default/default/upstream/7364998495670293450 {"id":7364998495670293450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table test.finish_mark not exists for 12-th check, retry later [2024/05/04 12:35:59.670 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:35:59.678 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:35:59.684 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] Create changefeed successfully! ID: 985a127f-38cc-4696-9a64-cf99c501da4b Info: {"upstream_id":7364998495670293450,"namespace":"default","id":"985a127f-38cc-4696-9a64-cf99c501da4b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:35:59.638636265+08:00","start_ts":449523838169120769,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523838169120769,"checkpoint_ts":449523838169120769,"checkpoint_time":"2024-05-04 12:35:56.297"} [2024/05/04 12:35:59.776 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:35:59.878 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:35:59.881 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:00.066 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:36:00.273 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:00.282 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:36:00.288 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:00.375 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:36:00.383 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:00.391 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:00.476 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:00.481 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] owner pid: 4416 owner id 43ede0f7-2ae0-474f-9692-1802786dfacc [Sat May 4 12:36:00 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.45514553.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/04 12:36:00.666 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:00.681 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:00.690 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:00.771 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:36:00.781 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] table test.t2 not exists for 2-th check, retry later check diff failed 3-th time, retry later [2024/05/04 12:36:00.872 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:00.895 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:36:00.983 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] table cdc_tiflash_test.multi_data_type exists [2024/05/04 12:36:01.170 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:01.225 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark20000] [2024/05/04 12:36:01.244 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark20000] [ts=449523839279562757] [2024/05/04 12:36:01.244 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark20000] [2024/05/04 12:36:01.244 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449523839279562757] [2024/05/04 12:36:01.247 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:36:01.250 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:36:01.252 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/04 12:36:01.256 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/04 12:36:01.257 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/04 12:36:01.257 +08:00] [INFO] [case.go:615] ["bank test finished"] [valid=20] [tries=20] [ratio=1] + cleanup_process cdc.test check diff failed 1-th time, retry later [2024/05/04 12:36:01.368 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:36:01.471 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:01.476 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:01.482 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:01.489 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table test.finish_mark not exists for 13-th check, retry later [2024/05/04 12:36:01.775 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:01.789 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] wait process cdc.test exit for 1-th time... [2024/05/04 12:36:01.872 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:01.891 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:01.993 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:02.066 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:36:02.067 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:36:02.097 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:02.185 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:36:02.197 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:02.266 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:36:02.273 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] wait process cdc.test exit for 2-th time... [2024/05/04 12:36:02.568 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] table test.t2 not exists for 3-th check, retry later table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later check diff failed 4-th time, retry later [2024/05/04 12:36:02.667 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:02.770 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] wait process cdc.test exit for 3-th time... [2024/05/04 12:36:02.867 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:36:02.895 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:36:02.984 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:03.069 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] check diff successfully [2024/05/04 12:36:03.181 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb0245c51_48de_41d0_9308_4023fed777bf"] [2024/05/04 12:36:03.184 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:36:03.187 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse003fdd9_0418_4cdc_9f7a_a51f1ea48b5a"] [2024/05/04 12:36:03.298 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] ./t/date_formats.test: ok! 99 test cases passed, take time 5.289754027 s time="2024-05-04T12:36:02+08:00" level=info msg="run test [date_formats] ok" time="2024-05-04T12:36:02+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0003a6378 {{} {} 0} {0 0} [0xc0004ac1b0] map[] 0 1 0xc00039f7a0 false map[0xc0004ac1b0:map[0xc0004ac1b0:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ++ date + echo '[Sat May 4 12:36:03 CST 2024] <<<<<< run test case bank success! >>>>>>' [Sat May 4 12:36:03 CST 2024] <<<<<< run test case bank success! >>>>>> + stop_tidb_cluster [2024/05/04 12:36:03.381 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/04 12:36:03.390 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4e5d232e_95dd_4a99_bf30_b9ce8247d65d"] [2024/05/04 12:36:03.393 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/04 12:36:03.583 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_test-1731/tiflow-cdc) 3730337792 bytes in 16.50 secs (226080715 bytes/sec) [Pipeline] { [2024/05/04 12:36:03.880 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:36:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031be1b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523839820890117 MinTableBarrierTs:449523840083034119 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43ede0f7-2ae0-474f-9692-1802786dfacc {"id":"43ede0f7-2ae0-474f-9692-1802786dfacc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797351} /tidb/cdc/default/__cdc_meta__/capture/cdb36027-bd55-4f86-9f85-18691cadbc0a {"id":"cdb36027-bd55-4f86-9f85-18691cadbc0a","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797360} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d21f 43ede0f7-2ae0-474f-9692-1802786dfacc /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d280 cdb36027-bd55-4f86-9f85-18691cadbc0a /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523839820890117,"min-table-barrier-ts":449523840083034119,"admin-job-type":0} /tidb/cdc/default/default/task/position/43ede0f7-2ae0-474f-9692-1802786dfacc/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cdb36027-bd55-4f86-9f85-18691cadbc0a/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031be1b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523839820890117 MinTableBarrierTs:449523840083034119 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43ede0f7-2ae0-474f-9692-1802786dfacc {"id":"43ede0f7-2ae0-474f-9692-1802786dfacc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797351} /tidb/cdc/default/__cdc_meta__/capture/cdb36027-bd55-4f86-9f85-18691cadbc0a {"id":"cdb36027-bd55-4f86-9f85-18691cadbc0a","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797360} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d21f 43ede0f7-2ae0-474f-9692-1802786dfacc /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d280 cdb36027-bd55-4f86-9f85-18691cadbc0a /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523839820890117,"min-table-barrier-ts":449523840083034119,"admin-job-type":0} /tidb/cdc/default/default/task/position/43ede0f7-2ae0-474f-9692-1802786dfacc/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/task/position/cdb36027-bd55-4f86-9f85-18691cadbc0a/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031be1b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523839820890117 MinTableBarrierTs:449523840083034119 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43ede0f7-2ae0-474f-9692-1802786dfacc {"id":"43ede0f7-2ae0-474f-9692-1802786dfacc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797351} /tidb/cdc/default/__cdc_meta__/capture/cdb36027-bd55-4f86-9f85-18691cadbc0a {"id":"cdb36027-bd55-4f86-9f85-18691cadbc0a","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797360} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d21f 43ede0f7-2ae0-474f-9692-1802786dfacc /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d280 cdb36027-bd55-4f86-9f85-18691cadbc0a /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523839820890117,"min-table-barrier-ts":449523840083034119,"admin-job-type":0} /tidb/cdc/default/default/task/position/43ede0f7-2ae0-474f-9692-1802786dfacc/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cdb36027-bd55-4f86-9f85-18691cadbc0a/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "43ede0f7-2ae0-474f-9692-1802786dfacc" | grep id wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 14-th check, retry later [2024/05/04 12:36:04.066 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:04.067 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa6a6d19e_886a_43a9_a898_5f9af4d36950"] [2024/05/04 12:36:04.070 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:04.076 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:04.175 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:36:04.179 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:36:04.287 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLscefdb5bf_8536_4735_89ec_96fe15fd0191"] [2024/05/04 12:36:04.370 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:04.379 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:04.392 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] wait process cdc.test exit for 2-th time... table test.t2 not exists for 4-th check, retry later [2024/05/04 12:36:04.480 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:04.673 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later check diff failed 5-th time, retry later [2024/05/04 12:36:04.776 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:36:04.778 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:36:04.779 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:04.781 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:36:04.881 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:36:04.885 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:36:04 CST 2024] <<<<<< run test case tiflash success! >>>>>> [2024/05/04 12:36:05.081 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:05.173 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:36:05.186 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] "cluster-id": "default" "id": "cdb36027-bd55-4f86-9f85-18691cadbc0a", "cluster-id": "default" run task successfully [2024/05/04 12:36:05.270 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:36:05.290 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:36:05.371 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] check_changefeed_state http://127.0.0.1:2379 2f6df294-032c-425d-900f-f94fc9ec1bc0 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=2f6df294-032c-425d-900f-f94fc9ec1bc0 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 2f6df294-032c-425d-900f-f94fc9ec1bc0 -s + info='{ "upstream_id": 7364998093334425536, "namespace": "default", "id": "2f6df294-032c-425d-900f-f94fc9ec1bc0", "state": "finished", "checkpoint_tso": 449523837499604997, "checkpoint_time": "2024-05-04 12:35:53.743", "error": null }' + echo '{ "upstream_id": 7364998093334425536, "namespace": "default", "id": "2f6df294-032c-425d-900f-f94fc9ec1bc0", "state": "finished", "checkpoint_tso": 449523837499604997, "checkpoint_time": "2024-05-04 12:35:53.743", "error": null }' { "upstream_id": 7364998093334425536, "namespace": "default", "id": "2f6df294-032c-425d-900f-f94fc9ec1bc0", "state": "finished", "checkpoint_tso": 449523837499604997, "checkpoint_time": "2024-05-04 12:35:53.743", "error": null } ++ echo '{' '"upstream_id":' 7364998093334425536, '"namespace":' '"default",' '"id":' '"2f6df294-032c-425d-900f-f94fc9ec1bc0",' '"state":' '"finished",' '"checkpoint_tso":' 449523837499604997, '"checkpoint_time":' '"2024-05-04' '12:35:53.743",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7364998093334425536, '"namespace":' '"default",' '"id":' '"2f6df294-032c-425d-900f-f94fc9ec1bc0",' '"state":' '"finished",' '"checkpoint_tso":' 449523837499604997, '"checkpoint_time":' '"2024-05-04' '12:35:53.743",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:36:00 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> [2024/05/04 12:36:05.583 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsced1f705_8bda_4ff6_af66_a97961cf72ab"] [2024/05/04 12:36:05.590 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:36:05.593 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:36:05.595 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:36:05.596 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:36:05.698 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:05.702 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:05.789 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:36:05.793 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:36:05.881 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/04 12:36:05.885 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table test.finish_mark exists check diff successfully [2024/05/04 12:36:05.985 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:05.994 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] wait process cdc.test exit for 1-th time... table test.t2 not exists for 5-th check, retry later [2024/05/04 12:36:06.469 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:06.469 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:06.483 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:36:06.484 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:06.485 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [Pipeline] sh table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later capture_id: cdb36027-bd55-4f86-9f85-18691cadbc0a [2024/05/04 12:36:06.575 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:06.685 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:36:06.689 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:36:06.767 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] wait process cdc.test exit for 2-th time... [2024/05/04 12:36:06.786 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:06.795 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:06.971 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G19 Run cases: changefeed_fast_fail batch_update_to_no_batch changefeed_resume_with_checkpoint_ts PROW_JOB_ID=fa5e2f5f-9800-4c28-be8c-afd6a4c48833 JENKINS_NODE_COOKIE=757611a6-ee9c-4e0a-bf6f-7cff079b07e5 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_test-1731 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_test","buildid":"1786610665142620166","prowjobid":"fa5e2f5f-9800-4c28-be8c-afd6a4c48833","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1731 TEST_GROUP=G19 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786610665142620166 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/1731/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_test_1731-bxqzg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-abe98160c26b91d7ad4012fc504dcc1b24097e77dc15bbc9f9f12631da13fdbf NODE_LABELS=pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms pingcap_tiflow_pull_cdc_integration_test_1731-bxqzg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=1731 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_fast_fail/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 6-th time, retry later [2024/05/04 12:36:07.195 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:36:07.197 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:36:07.197 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:36:07.277 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:36:07.280 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] pass check, checkpoint tso not forward after 10s run task successfully wait process 8093 exit for 1-th time... wait process 8093 exit for 2-th time... wait process 8093 exit for 3-th time... wait process 8093 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8093) - No such process wait process 8093 exit for 5-th time... process 8093 already exit [Sat May 4 12:36:00 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.87878789.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:36:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364998164698770627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:42.248007212 +0800 CST StartTs:449523818735599617 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0041a5e60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523818748706823} {CheckpointTs:449523822615330828 MinTableBarrierTs:449523839864406019 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2a11589-57e6-4dbf-9f8f-b40ce8310523 {"id":"d2a11589-57e6-4dbf-9f8f-b40ce8310523","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797361} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b99bb1 d2a11589-57e6-4dbf-9f8f-b40ce8310523 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364998164698770627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:34:42.248007212+08:00","start-ts":449523818735599617,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523818748706823} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449523822615330828,"min-table-barrier-ts":449523839864406019,"admin-job-type":0} /tidb/cdc/default/default/task/position/d2a11589-57e6-4dbf-9f8f-b40ce8310523/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998164698770627 {"id":7364998164698770627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364998164698770627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:42.248007212 +0800 CST StartTs:449523818735599617 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0041a5e60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523818748706823} {CheckpointTs:449523822615330828 MinTableBarrierTs:449523839864406019 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2a11589-57e6-4dbf-9f8f-b40ce8310523 {"id":"d2a11589-57e6-4dbf-9f8f-b40ce8310523","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797361} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b99bb1 d2a11589-57e6-4dbf-9f8f-b40ce8310523 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364998164698770627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:34:42.248007212+08:00","start-ts":449523818735599617,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523818748706823} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449523822615330828,"min-table-barrier-ts":449523839864406019,"admin-job-type":0} /tidb/cdc/default/default/task/position/d2a11589-57e6-4dbf-9f8f-b40ce8310523/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998164698770627 {"id":7364998164698770627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364998164698770627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:42.248007212 +0800 CST StartTs:449523818735599617 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0041a5e60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523818748706823} {CheckpointTs:449523822615330828 MinTableBarrierTs:449523839864406019 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449523822615330828, checkpointTs: 449523822615330828, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2a11589-57e6-4dbf-9f8f-b40ce8310523 {"id":"d2a11589-57e6-4dbf-9f8f-b40ce8310523","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797361} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e1b99bb1 d2a11589-57e6-4dbf-9f8f-b40ce8310523 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364998164698770627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-04T12:34:42.248007212+08:00","start-ts":449523818735599617,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523818748706823} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449523822615330828,"min-table-barrier-ts":449523839864406019,"admin-job-type":0} /tidb/cdc/default/default/task/position/d2a11589-57e6-4dbf-9f8f-b40ce8310523/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998164698770627 {"id":7364998164698770627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff successfully check_ts_forward ddl-only-block-related-table changefeed is working normally rts: 449523822615330800->449523840388694000 checkpoint: 449523822615330800->449523840428015600 run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:36:07 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> [2024/05/04 12:36:07.283 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:07.289 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs440187a2_1ea4_4362_a045_d2ef77611780"] [2024/05/04 12:36:07.489 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:36:07.571 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:36:07.584 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:07.768 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:36:07.772 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:36:07 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> Click here to forcibly terminate running steps [2024/05/04 12:36:07.982 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:36:08.182 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsbb71406c_7ba3_4cdd_b6a4_548b0c02145a"] [2024/05/04 12:36:08.189 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:36:08.189 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:08.274 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:08.278 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:36:08.279 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:08.291 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:36:08.369 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:36:08.380 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:08.471 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:36:08.472 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:36:08.478 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:36:08.666 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:36:08.685 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table test.t2 not exists for 6-th check, retry later check diff failed 2-th time, retry later check diff successfully [2024/05/04 12:36:09.270 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["consistent_replicate_nfs.usertable*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449523837980901512" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/04 12:36:09.479 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:09.483 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:09.487 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:09.574 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:09.579 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:36:09.584 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:09.499 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/04 12:36:09.500 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2024/05/04 12:36:09.500 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=:6060] [2024/05/04 12:36:09.500 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/04 12:36:09.500 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=449523837404184691] [resolvedTs=449523837980901512] [2024/05/04 12:36:09.501 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=219.635Β΅s] [2024/05/04 12:36:09.503 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 12:36:09.503 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 12:36:09.503 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 12:36:09.508 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 12:36:09.511 +08:00] [INFO] [mysql.go:197] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/04 12:36:09.513 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/04 12:36:09.513 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/04 12:36:09.513 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/04 12:36:09.512 +08:00] [INFO] [worker.go:86] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/04 12:36:09.513 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/04 12:36:09.513 +08:00] [WARN] [config.go:395] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/04 12:36:09.513 +08:00] [ERROR] [config.go:210] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [default=0] [2024/05/04 12:36:09.516 +08:00] [INFO] [db_helper.go:173] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/04 12:36:09.518 +08:00] [INFO] [mysql_ddl_sink.go:107] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/04 12:36:09.521 +08:00] [INFO] [file.go:284] ["ignore logs which commitTs is greater than resolvedTs"] [filename=127.0.0.1:8300_a15236b7-9adc-45e9-8e43-3b3aa2b7cd8a_row_449523837980901513_8b2b7584-8936-4f96-bd90-20f4efe9ceb3.log] [endTs=449523837980901512] [2024/05/04 12:36:09.523 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=22.439253ms] nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/04 12:36:09.681 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:36:09.684 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:36:09.691 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:36:09.771 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:09.772 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:36:09.869 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:36:09.870 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:36:09.872 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:09.767 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}] [checkpointTs=449523837980901512] [2024/05/04 12:36:09.767 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}] [checkpointTs=449523837980901512] [2024/05/04 12:36:09.867 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=2994] [appliedDDLCount=0] [currentCheckpoint=449523837980901512] [2024/05/04 12:36:09.867 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/04 12:36:09.867 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/04 12:36:09.867 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/04 12:36:09.868 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/04 12:36:09.867 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/04 12:36:09.868 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/04 12:36:09.867 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/04 12:36:09.868 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/04 12:36:09.868 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/04 12:36:09.868 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/04 12:36:09.868 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/04 12:36:09.868 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/04 12:36:09.868 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/04 12:36:09.868 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/04 12:36:09.868 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/04 12:36:09.868 +08:00] [INFO] [worker.go:94] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] Apply redo log successfully [2024/05/04 12:36:09.985 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] check diff successfully [Sat May 4 12:36:10 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> count(*) 47806 start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 12:36:10.382 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:36:10.576 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:36:10.586 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:36:10.588 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table test.t2 not exists for 7-th check, retry later [2024/05/04 12:36:10.673 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:10.682 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:10.776 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:36:10.870 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:36:10.880 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:10.887 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] check diff failed 3-th time, retry later [2024/05/04 12:36:10.971 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:10.976 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:36:10.980 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:10.988 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:11.066 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:11.167 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:36:11.482 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:11.583 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:36:11.675 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:11.678 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:11.683 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/04 12:36:11.775 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:11.884 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:11.977 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:11.993 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:12.077 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:36:12.083 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:12.181 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:36:12.265 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:12.272 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:12.284 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:12.374 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:12.581 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:12.584 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:12.668 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:12.672 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:12.673 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:12.680 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table test.t2 not exists for 8-th check, retry later check diff failed 4-th time, retry later [2024/05/04 12:36:12.980 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:36:12.986 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:36:13.081 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:13.182 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:13.277 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] ./t/drop.test: ok! 48 test cases passed, take time 8.706053621 s time="2024-05-04T12:36:11+08:00" level=info msg="run test [drop] ok" time="2024-05-04T12:36:11+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000126118 {{} {} 0} {0 0} [0xc00030a000] map[] 0 1 0xc0000367e0 false map[0xc00030a000:map[0xc00030a000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" ./t/mysql_replace.test: ok! 16 test cases passed, take time 1.043781258 s time="2024-05-04T12:36:13+08:00" level=info msg="run test [mysql_replace] ok" Great, All tests passed mysqltest end [2024/05/04 12:36:13.376 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:36:13.386 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:13.474 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/kill_owner_with_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/04 12:36:13.568 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:36:13.582 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:13.786 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] table test.finish_mark not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 12:36:13.868 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:13.869 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:13.871 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:13.877 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:13.978 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/04 12:36:14.085 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:14.279 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:14.280 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/04 12:36:14.379 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:14.573 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] table test.t2 not exists for 9-th check, retry later [2024/05/04 12:36:14.587 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:36:14.673 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:36:14.678 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:36:14.690 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:36:14.694 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/04 12:36:14.865 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:14.869 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:36:14.872 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:14.873 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:36:14.876 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:14.897 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:36:14.985 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:15.170 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:15.184 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:36:15.189 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] check diff failed 5-th time, retry later [2024/05/04 12:36:15.566 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:15.587 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] Starting Upstream TiDB... [2024/05/04 12:36:15.670 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:36:15.682 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:15.690 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:15.691 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:15.767 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:15.768 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:15.771 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:36:15.772 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:36:15.774 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:36:15.872 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:15.977 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:36:15.984 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:16.065 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 2-th time, retry later table test.finish_mark not exists for 2-th check, retry later [2024/05/04 12:36:16.188 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/04 12:36:13.017 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 11.771825133s"] [2024/05/04 12:36:16.585 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] table test.t2 not exists for 10-th check, retry later [2024/05/04 12:36:16.689 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:16.773 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:36:16.773 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:16.774 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:16.782 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:16.790 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:36:16.790 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:16.871 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:16.874 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:16.965 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:16.971 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:36:16.992 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:36:17.091 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] check diff failed 6-th time, retry later [2024/05/04 12:36:17.371 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:36:17.566 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:36:17.568 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:17.572 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:36:17.573 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:17.582 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:36:17.670 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:17.670 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:36:17.693 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:36:17.766 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/04 12:36:17.785 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/04 12:36:17.881 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 3-th time, retry later [2024/05/04 12:36:18.171 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table test.finish_mark not exists for 3-th check, retry later [2024/05/04 12:36:18.271 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:18.271 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:18.372 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:36:18.376 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] table test.t2 not exists for 11-th check, retry later [2024/05/04 12:36:18.474 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/04 12:36:18.475 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/04 12:36:18.477 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/04 12:36:18.679 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/04 12:36:18.807 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/04 12:36:18.895 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:18.895 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully [2024/05/04 12:36:18.974 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsfcfea499_e28f_4d43_8d00_5b939036c91c"] [2024/05/04 12:36:18.990 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:36:18.991 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] check diff failed 1-th time, retry later [2024/05/04 12:36:19.377 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/04 12:36:19.566 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/04 12:36:19.572 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/04 12:36:19.573 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/04 12:36:19.589 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa005ebdf_878f_4512_b501_e036ae114f20"] [2024/05/04 12:36:19.590 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:19.599 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:19.601 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:36:19.614 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/04 12:36:19.886 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] check diff failed 4-th time, retry later table test.finish_mark not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/04 12:36:20.100 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs88c9589e_abf1_4684_9bb0_2f60f67c04cc"] [2024/05/04 12:36:20.180 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/04 12:36:20.195 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:36:20.268 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:20.269 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:36:20.272 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:36:20.279 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/04 12:36:20.283 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:36:20.776 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:20.777 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:36:20.807 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:36:20.811 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:36:20.811 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:36:20.824 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:20.879 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse3dce3d3_781a_434b_8d89_eca41d6c1beb"] table test.t2 not exists for 12-th check, retry later check diff successfully start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... table processor_delay.t1 exists table processor_delay.t2 exists table processor_delay.t3 exists table processor_delay.t4 exists table processor_delay.t5 exists table processor_delay.t6 exists table processor_delay.t7 exists table processor_delay.t8 exists table processor_delay.t9 exists table processor_delay.t10 exists table processor_delay.t11 exists table processor_delay.t12 exists table processor_delay.t13 not exists for 1-th check, retry later table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later [2024/05/04 12:36:21.378 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:36:21.390 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:36:21.473 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:36:21.480 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:21.485 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:21.505 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:36:21.576 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:21.583 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7826f0bc_995f_49cf_8310_f52d881b8723"] [2024/05/04 12:36:21.588 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table clustered_index_test.t0 exists table clustered_index_test.t1 exists table clustered_index_test.t2 exists check table exists success check diff failed 1-th time, retry later check diff failed 2-th time, retry later wait process cdc.test exit for 1-th time... table processor_delay.t14 exists table processor_delay.t15 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:36:22.182 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e16240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms, pid:1289, start at 2024-05-04 12:36:21.040081547 +0800 CST m=+5.132445930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:21.046 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:21.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:21.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e16240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms, pid:1289, start at 2024-05-04 12:36:21.040081547 +0800 CST m=+5.132445930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:21.046 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:21.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:21.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e18680014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-bxqzg-nvfw0-ngkms, pid:1375, start at 2024-05-04 12:36:21.169826886 +0800 CST m=+5.194695577 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:21.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:21.146 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:21.146 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/04 12:36:22.285 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:22.377 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:22.385 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:36:22.395 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:36:22.409 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table test.finish_mark not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/04 12:36:22.466 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/04 12:36:22.479 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/04 12:36:22.481 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/04 12:36:22.486 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 exists check diff successfully [2024/05/04 12:36:22.775 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:22.794 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] wait process cdc.test exit for 3-th time... wait process cdc.test exit for 1-th time... [2024/05/04 12:36:22.968 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:36:22.975 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:22.983 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:36:23.070 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:36:23.076 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:36:23.087 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/04 12:36:23.092 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:23.092 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/04 12:36:23.373 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:36:23.390 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:36:23 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> [2024/05/04 12:36:23.579 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:23.581 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:23.597 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:36:23.689 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] wait process cdc.test exit for 2-th time... [2024/05/04 12:36:23.776 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:23.778 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/04 12:36:23.789 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/04 12:36:23.791 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/04 12:36:24.175 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:24.181 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2625.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 3-th time... nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 6-th check, retry later check diff failed 6-th time, retry later table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later [2024/05/04 12:36:24.375 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:24.375 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:24.388 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:36:24.479 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:24.492 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:36:24.497 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/04 12:36:24.498 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/04 12:36:24.506 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e47a40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp, pid:3992, start at 2024-05-04 12:36:24.21623854 +0800 CST m=+5.601331788 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:24.223 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:24.219 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:24.219 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 12:36:24 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> [2024/05/04 12:36:24.775 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:24.870 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:36:24.976 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:24.980 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:25.173 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:25.270 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:25.275 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/04 12:36:25.291 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/04 12:36:25.299 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:36:25.399 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:25.413 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:25.502 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:25.586 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:25.772 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/04 12:36:25.778 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/04 12:36:25.792 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/04 12:36:25.804 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/04 12:36:25.911 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:25.931 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:26.092 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:26.098 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] + set +x + tso='449523845575737345 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523845575737345 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:36:26 CST 2024] <<<<<< START cdc server in changefeed_fast_fail case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.26692671.out server --log-file /tmp/tidb_cdc_test/changefeed_fast_fail/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_fast_fail/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:36:26.172 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/04 12:36:26.221 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:26.246 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:26.295 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] table processor_delay.t16 exists table processor_delay.t17 not exists for 1-th check, retry later table test.finish_mark not exists for 7-th check, retry later check diff successfully [2024/05/04 12:36:26.377 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:26.384 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/04 12:36:26.475 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/04 12:36:26.523 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:26.568 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:26.613 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/04 12:36:26.697 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:26.702 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/04 12:36:26.786 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/04 12:36:26.834 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 12:36:26.996 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/04 12:36:27.000 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/04 12:36:27.102 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] check diff failed 5-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e47a40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp, pid:3992, start at 2024-05-04 12:36:24.21623854 +0800 CST m=+5.601331788 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:24.223 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:24.219 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:24.219 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e49380005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-x4lzj-z4m7f-17txp, pid:4079, start at 2024-05-04 12:36:24.276164227 +0800 CST m=+5.585031735 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:24.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:24.270 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:24.270 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/04 12:36:27.178 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/04 12:36:27.346 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 8-th check, retry later table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later [2024/05/04 12:36:28.008 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v1m14.435767565s"] [2024/05/04 12:36:28.518 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 1m14.946451973s"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) empty select id, val from test.availability3 where id=1 run task successfully test_gap_between_watch_capture pass wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... check diff failed 6-th time, retry later table mark.finish_mark_1 not exists for 1-th check, retry later [Sat May 4 12:36:29 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.55015503.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // container Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:36:29 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7e88678-c1fa-4e0e-b782-15baf137c34c {"id":"c7e88678-c1fa-4e0e-b782-15baf137c34c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e35bb8da c7e88678-c1fa-4e0e-b782-15baf137c34c /tidb/cdc/default/default/upstream/7364998634231680613 {"id":7364998634231680613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7e88678-c1fa-4e0e-b782-15baf137c34c {"id":"c7e88678-c1fa-4e0e-b782-15baf137c34c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e35bb8da c7e88678-c1fa-4e0e-b782-15baf137c34c /tidb/cdc/default/default/upstream/7364998634231680613 {"id":7364998634231680613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7e88678-c1fa-4e0e-b782-15baf137c34c {"id":"c7e88678-c1fa-4e0e-b782-15baf137c34c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797386} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e35bb8da c7e88678-c1fa-4e0e-b782-15baf137c34c /tidb/cdc/default/default/upstream/7364998634231680613 {"id":7364998634231680613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2728.out cli changefeed create --start-ts=449523845575737345 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-fast-fail [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case delete_owner_key [Sat May 4 12:36:29 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.49724974.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: changefeed-fast-fail Info: {"upstream_id":7364998634231680613,"namespace":"default","id":"changefeed-fast-fail","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T12:36:29.687552307+08:00","start_ts":449523845575737345,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523845575737345,"checkpoint_ts":449523845575737345,"checkpoint_time":"2024-05-04 12:36:24.551"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later table test.finish_mark not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e73740013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj, pid:7100, start at 2024-05-04 12:36:26.996161495 +0800 CST m=+5.330442756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:27.002 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:26.973 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:26.973 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e73740013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj, pid:7100, start at 2024-05-04 12:36:26.996161495 +0800 CST m=+5.330442756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:27.002 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:26.973 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:26.973 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e735c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-dm64w-5v4vw-5wnfj, pid:7180, start at 2024-05-04 12:36:27.01072725 +0800 CST m=+5.278569548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:27.017 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:27.018 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:27.018 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e86300013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r, pid:3772, start at 2024-05-04 12:36:28.219302419 +0800 CST m=+5.832267799 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:28.227 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:28.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:28.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e86300013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r, pid:3772, start at 2024-05-04 12:36:28.219302419 +0800 CST m=+5.832267799 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:28.227 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:28.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:28.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078e86140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-t604k-7wvcn-g3r4r, pid:3843, start at 2024-05-04 12:36:28.209442895 +0800 CST m=+5.772101512 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:28.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:28.216 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:28.216 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 7-th time, retry later table mark.finish_mark_1 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-fast-fail failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-fast-fail + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-fast-fail -s + info='{ "upstream_id": 7364998634231680613, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449523845575737345, "checkpoint_time": "2024-05-04 12:36:24.551", "error": { "time": "2024-05-04T12:36:29.748246518+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7364998634231680613, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449523845575737345, "checkpoint_time": "2024-05-04 12:36:24.551", "error": { "time": "2024-05-04T12:36:29.748246518+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7364998634231680613, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 449523845575737345, "checkpoint_time": "2024-05-04 12:36:24.551", "error": { "time": "2024-05-04T12:36:29.748246518+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7364998634231680613, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 449523845575737345, '"checkpoint_time":' '"2024-05-04' '12:36:24.551",' '"error":' '{' '"time":' '"2024-05-04T12:36:29.748246518+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7364998634231680613, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 449523845575737345, '"checkpoint_time":' '"2024-05-04' '12:36:24.551",' '"error":' '{' '"time":' '"2024-05-04T12:36:29.748246518+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully [Sat May 4 12:36:31 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.84428444.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:36:32 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2888eb1-5ae6-43aa-9a97-c830e501fb66 {"id":"b2888eb1-5ae6-43aa-9a97-c830e501fb66","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e36a3dcc b2888eb1-5ae6-43aa-9a97-c830e501fb66 /tidb/cdc/default/default/upstream/7364998648707383976 {"id":7364998648707383976,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2888eb1-5ae6-43aa-9a97-c830e501fb66 {"id":"b2888eb1-5ae6-43aa-9a97-c830e501fb66","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e36a3dcc b2888eb1-5ae6-43aa-9a97-c830e501fb66 /tidb/cdc/default/default/upstream/7364998648707383976 {"id":7364998648707383976,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2888eb1-5ae6-43aa-9a97-c830e501fb66 {"id":"b2888eb1-5ae6-43aa-9a97-c830e501fb66","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e36a3dcc b2888eb1-5ae6-43aa-9a97-c830e501fb66 /tidb/cdc/default/default/upstream/7364998648707383976 {"id":7364998648707383976,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 838478a3-f3ba-4379-bef1-61a514205716 Info: {"upstream_id":7364998648707383976,"namespace":"default","id":"838478a3-f3ba-4379-bef1-61a514205716","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-04T12:36:32.361965398+08:00","start_ts":449523846747258883,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523846747258883,"checkpoint_ts":449523846747258883,"checkpoint_time":"2024-05-04 12:36:29.020"} check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078eb3dc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh, pid:20016, start at 2024-05-04 12:36:31.138844719 +0800 CST m=+5.101818082 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:31.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:31.146 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:31.146 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078eb3dc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh, pid:20016, start at 2024-05-04 12:36:31.138844719 +0800 CST m=+5.101818082 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:31.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:31.146 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:31.146 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078eb4840014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-xjtcb-kc3v5-xn6fh, pid:20096, start at 2024-05-04 12:36:31.185955436 +0800 CST m=+5.063500056 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:31.192 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:31.187 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:31.187 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later table test.finish_mark not exists for 10-th check, retry later [Pipeline] // cache [Pipeline] } wait process cdc.test exit for 1-th time... table mark.finish_mark_1 not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:36:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004026cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523846374490119 MinTableBarrierTs:449523847685210119 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523846374490119, checkpointTs: 449523846374490119, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523846374490119, checkpointTs: 449523846374490119, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523846374490119, checkpointTs: 449523846374490119, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e87e81b7-11ea-4f24-8828-18028819be9d {"id":"e87e81b7-11ea-4f24-8828-18028819be9d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797390} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d389 e87e81b7-11ea-4f24-8828-18028819be9d /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523846374490119,"min-table-barrier-ts":449523847685210119,"admin-job-type":0} /tidb/cdc/default/default/task/position/e87e81b7-11ea-4f24-8828-18028819be9d/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004026cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523846374490119 MinTableBarrierTs:449523847685210119 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523846374490119, checkpointTs: 449523846374490119, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523846374490119, checkpointTs: 449523846374490119, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523846374490119, checkpointTs: 449523846374490119, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e87e81b7-11ea-4f24-8828-18028819be9d {"id":"e87e81b7-11ea-4f24-8828-18028819be9d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797390} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d389 e87e81b7-11ea-4f24-8828-18028819be9d /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523846374490119,"min-table-barrier-ts":449523847685210119,"+ grep -q 'failed to get info:' admin-job-type":0} /tidb/cdc/default/default/task/position/e87e81b7-11ea-4f24-8828-18028819be9d/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/361cab25-2b13-4166-88e2-41e4aa8a9624 {UpstreamID:7364998079142423954 Namespace:default ID:361cab25-2b13-4166-88e2-41e4aa8a9624 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-04 12:34:23.420688458 +0800 CST StartTs:449523812426842113 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004026cf0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-57-g20ef19816 Epoch:449523813816205316} {CheckpointTs:449523846374490119 MinTableBarrierTs:449523847685210119 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449523846374490119, checkpointTs: 449523846374490119, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449523846374490119, checkpointTs: 449523846374490119, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449523846374490119, checkpointTs: 449523846374490119, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e87e81b7-11ea-4f24-8828-18028819be9d {"id":"e87e81b7-11ea-4f24-8828-18028819be9d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797390} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d389 e87e81b7-11ea-4f24-8828-18028819be9d /tidb/cdc/default/default/changefeed/info/361cab25-2b13-4166-88e2-41e4aa8a9624 {"upstream-id":7364998079142423954,"namespace":"default","changefeed-id":"361cab25-2b13-4166-88e2-41e4aa8a9624","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-04T12:34:23.420688458+08:00","start-ts":449523812426842113,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-57-g20ef19816","epoch":449523813816205316} /tidb/cdc/default/default/changefeed/status/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":449523846374490119,"min-table-barrier-ts":449523847685210119,"admin-job-type":0} /tidb/cdc/default/default/task/position/e87e81b7-11ea-4f24-8828-18028819be9d/361cab25-2b13-4166-88e2-41e4aa8a9624 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364998079142423954 {"id":7364998079142423954,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' wait process cdc.test exit for 2-th time... [Sat May 4 12:36:32 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.51275129.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 8-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 12:36:33 CST 2024] <<<<<< run test case changefeed_fast_fail success! >>>>>> table force_replicate_table.t0 not exists for 1-th check, retry later "is-owner": true, run task successfully table test.finish_mark not exists for 11-th check, retry later table processor_delay.t20 exists table processor_delay.t21 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078edddc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:9488, start at 2024-05-04 12:36:33.818567069 +0800 CST m=+5.640195082 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:33.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:33.833 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:33.833 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 4-th check, retry later check diff failed 3-th time, retry later check diff failed 9-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078ee6b80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:9975, start at 2024-05-04 12:36:34.396650266 +0800 CST m=+5.379671323 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:34.405 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:34.400 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:34.400 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:36:35 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5087f88e-9dcc-4245-be35-cd4368931df7 {"id":"5087f88e-9dcc-4245-be35-cd4368931df7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797392} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e37244cc 5087f88e-9dcc-4245-be35-cd4368931df7 /tidb/cdc/default/default/upstream/7364998661329262936 {"id":7364998661329262936,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5087f88e-9dcc-4245-be35-cd4368931df7 {"id":"5087f88e-9dcc-4245-be35-cd4368931df7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797392} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e37244cc 5087f88e-9dcc-4245-be35-cd4368931df7 /tidb/cdc/default/default/upstream/7364998661329262936 {"id":7364998661329262936,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5087f88e-9dcc-4245-be35-cd4368931df7 {"id":"5087f88e-9dcc-4245-be35-cd4368931df7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797392} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e37244cc 5087f88e-9dcc-4245-be35-cd4368931df7 /tidb/cdc/default/default/upstream/7364998661329262936 {"id":7364998661329262936,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 5f9b1c29-35bb-4171-9f4c-6ad40f3b2f8b Info: {"upstream_id":7364998661329262936,"namespace":"default","id":"5f9b1c29-35bb-4171-9f4c-6ad40f3b2f8b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T12:36:35.250904773+08:00","start_ts":449523848347385860,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523848347385860,"checkpoint_ts":449523848347385860,"checkpoint_time":"2024-05-04 12:36:35.124"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.21426.out cli tso query --pd=http://127.0.0.1:2379 owner pid: 4977 owner id e87e81b7-11ea-4f24-8828-18028819be9d default owner key /tidb/cdc/default/__cdc_meta__/owner/22318f41e163d389 [Sat May 4 12:36:35 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.51115113.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:36:35 GMT < Content-Length: 853 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4cbf4a99-9b2e-4daa-82fe-45734b28dbf5 {"id":"4cbf4a99-9b2e-4daa-82fe-45734b28dbf5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797393} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e378e392 4cbf4a99-9b2e-4daa-82fe-45734b28dbf5 /tidb/cdc/default/default/upstream/7364998663054083755 {"id":7364998663054083755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4cbf4a99-9b2e-4daa-82fe-45734b28dbf5 {"id":"4cbf4a99-9b2e-4daa-82fe-45734b28dbf5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797393} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e378e392 4cbf4a99-9b2e-4daa-82fe-45734b28dbf5 /tidb/cdc/default/default/upstream/7364998663054083755 {"id":7364998663054083755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4cbf4a99-9b2e-4daa-82fe-45734b28dbf5 {"id":"4cbf4a99-9b2e-4daa-82fe-45734b28dbf5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797393} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e378e392 4cbf4a99-9b2e-4daa-82fe-45734b28dbf5 /tidb/cdc/default/default/upstream/7364998663054083755 {"id":7364998663054083755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: ed21f0e9-8896-47fc-809f-72232740c4c8 Info: {"upstream_id":7364998663054083755,"namespace":"default","id":"ed21f0e9-8896-47fc-809f-72232740c4c8","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T12:36:36.013416035+08:00","start_ts":449523848543469572,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-57-g20ef19816","resolved_ts":449523848543469572,"checkpoint_ts":449523848543469572,"checkpoint_time":"2024-05-04 12:36:35.872"} start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table force_replicate_table.t0 exists table force_replicate_table.t1 not exists for 1-th check, retry later table test.finish_mark not exists for 12-th check, retry later table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later table kill_owner_with_ddl.t1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078edddc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:9488, start at 2024-05-04 12:36:33.818567069 +0800 CST m=+5.640195082 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:33.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:33.833 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:33.833 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078edf700013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-l3kc8-cjn6b-j990l, pid:9565, start at 2024-05-04 12:36:33.930332492 +0800 CST m=+5.701056578 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:33.940 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:33.933 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:33.933 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 4-th time, retry later Sending interrupt signal to process Killing processes check diff failed 10-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078ee6b80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:9975, start at 2024-05-04 12:36:34.396650266 +0800 CST m=+5.379671323 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:34.405 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:34.400 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:34.400 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d078ee8340016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-test-1731-lq24j-sdjs3-qr61t, pid:10049, start at 2024-05-04 12:36:34.475610304 +0800 CST m=+5.406961917 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-12:38:34.482 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-12:36:34.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-12:26:34.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } split_and_random_merge scale: 20 table mark.finish_mark_1 not exists for 5-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 + set +x + tso='449523848575713281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449523848575713281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 12:36:37 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2147621478.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 script returned exit code 143 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release kill finished with exit code 0 Sending interrupt signal to process Killing processes table force_replicate_table.t1 exists table force_replicate_table.t2 not exists for 1-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later {"level":"warn","ts":1714797398.2431767,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003dc2a80/127.0.0.1:2479","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes [Sat May 4 12:36:38 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1088710889.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 {"level":"warn","ts":1714797398.610215,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001fde1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 check diff failed 5-th time, retry later script returned exit code 143 Verifying downstream PD is started... table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "requestdistribution"="uniform" "threadcount"="4" "workload"="core" "mysql.db"="changefeed_auto_stop_1" "readproportion"="0" "mysql.user"="root" "dotransactions"="false" "insertproportion"="0" "recordcount"="20" "updateproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.port"="4000" "operationcount"="0" ********************************************** Run finished, takes 10.506679ms INSERT - Takes(s): 0.0, Count: 19, OPS: 3310.4, Avg(us): 1819, Min(us): 1026, Max(us): 5124, 95th(us): 6000, 99th(us): 6000 ***************** properties ***************** "mysql.db"="changefeed_auto_stop_2" "dotransactions"="false" "threadcount"="4" "mysql.user"="root" "updateproportion"="0" "recordcount"="20" "mysql.port"="4000" "insertproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "readproportion"="0" "workload"="core" "mysql.host"="127.0.0.1" "readallfields"="true" "scanproportion"="0" ********************************************** Run finished, takes 10.369211ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3224.9, Avg(us): 1911, Min(us): 1000, Max(us): 4534, 95th(us): 5000, 99th(us): 5000 table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later ***************** properties ***************** "mysql.port"="4000" "operationcount"="0" "readallfields"="true" "readproportion"="0" "threadcount"="4" "mysql.user"="root" "requestdistribution"="uniform" "workload"="core" "recordcount"="20" "dotransactions"="false" "updateproportion"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.db"="changefeed_auto_stop_3" "insertproportion"="0" ********************************************** Run finished, takes 9.503349ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3523.8, Avg(us): 1767, Min(us): 1027, Max(us): 4357, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "recordcount"="20" "threadcount"="4" "dotransactions"="false" "requestdistribution"="uniform" "readproportion"="0" "mysql.port"="4000" "updateproportion"="0" "scanproportion"="0" "mysql.user"="root" "workload"="core" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_auto_stop_4" "operationcount"="0" "insertproportion"="0" "readallfields"="true" ********************************************** Run finished, takes 9.219491ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3475.7, Avg(us): 1694, Min(us): 987, Max(us): 3691, 95th(us): 4000, 99th(us): 4000 [Sat May 4 12:36:40 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1150211504.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 6-th time, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes table processor_delay.t23 exists {"level":"warn","ts":1714797400.7500677,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001778a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} table processor_delay.t24 not exists for 1-th check, retry later script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:36:40 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/636affb2-8410-432b-a840-c068f78586ed {"id":"636affb2-8410-432b-a840-c068f78586ed","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797397} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e38740ed 636affb2-8410-432b-a840-c068f78586ed /tidb/cdc/default/default/upstream/7364998676196355391 {"id":7364998676196355391,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/636affb2-8410-432b-a840-c068f78586ed {"id":"636affb2-8410-432b-a840-c068f78586ed","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797397} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e38740ed 636affb2-8410-432b-a840-c068f78586ed /tidb/cdc/default/default/upstream/7364998676196355391 {"id":7364998676196355391,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/636affb2-8410-432b-a840-c068f78586ed {"id":"636affb2-8410-432b-a840-c068f78586ed","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797397} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e38740ed 636affb2-8410-432b-a840-c068f78586ed /tidb/cdc/default/default/upstream/7364998676196355391 {"id":7364998676196355391,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table mark.finish_mark_2 not exists for 2-th check, retry later script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714797401.3831506,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002228e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:36:41 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3029fc9-2a6c-4d37-a4e6-50ec4a6e8ac4 {"id":"f3029fc9-2a6c-4d37-a4e6-50ec4a6e8ac4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797398} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e38da5cb f3029fc9-2a6c-4d37-a4e6-50ec4a6e8ac4 /tidb/cdc/default/default/upstream/7364998689560362536 {"id":7364998689560362536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3029fc9-2a6c-4d37-a4e6-50ec4a6e8ac4 {"id":"f3029fc9-2a6c-4d37-a4e6-50ec4a6e8ac4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797398} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e38da5cb f3029fc9-2a6c-4d37-a4e6-50ec4a6e8ac4 /tidb/cdc/default/default/upstream/7364998689560362536 {"id":7364998689560362536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3029fc9-2a6c-4d37-a4e6-50ec4a6e8ac4 {"id":"f3029fc9-2a6c-4d37-a4e6-50ec4a6e8ac4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797398} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e38da5cb f3029fc9-2a6c-4d37-a4e6-50ec4a6e8ac4 /tidb/cdc/default/default/upstream/7364998689560362536 {"id":7364998689560362536,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x kill finished with exit code 0 Sending interrupt signal to process Killing processes lease 22318f41e38740ed revoked {"level":"warn","ts":1714797401.923622,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0023a5180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 ***************** properties ***************** "mysql.host"="127.0.0.1" "operationcount"="0" "readproportion"="0" "mysql.user"="root" "readallfields"="true" "dotransactions"="false" "insertproportion"="0" "requestdistribution"="uniform" "mysql.port"="4000" "scanproportion"="0" "fieldcount"="100" "threadcount"="10" "recordcount"="5000" "updateproportion"="0" "mysql.db"="consistent_replicate_storage_file" "workload"="core" ********************************************** kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 {"level":"warn","ts":1714797402.4652393,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002348fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 {"level":"warn","ts":1714797401.6545706,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002473500/127.0.0.1:2779","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes [2024/05/04 12:36:40.564 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column name not equal, one is Oe1LE another is c"] script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 04:36:43 GMT < Content-Length: 809 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21bd0e36-f7a9-4a5e-a9c7-6bc8d4e1587b {"id":"21bd0e36-f7a9-4a5e-a9c7-6bc8d4e1587b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e3920135 21bd0e36-f7a9-4a5e-a9c7-6bc8d4e1587b /tidb/cdc/default/default/upstream/7364998688785556323 {"id":7364998688785556323,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21bd0e36-f7a9-4a5e-a9c7-6bc8d4e1587b {"id":"21bd0e36-f7a9-4a5e-a9c7-6bc8d4e1587b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e3920135 21bd0e36-f7a9-4a5e-a9c7-6bc8d4e1587b /tidb/cdc/default/default/upstream/7364998688785556323 {"id":7364998688785556323,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21bd0e36-f7a9-4a5e-a9c7-6bc8d4e1587b {"id":"21bd0e36-f7a9-4a5e-a9c7-6bc8d4e1587b","address":"127.0.0.1:8301","version":"v8.2.0-alpha-57-g20ef19816","git-hash":"20ef198164f6ae8a5061b6b8ab1338f31dc54a4a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_test/tiflow/bin/cdc.test","start-timestamp":1714797400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41e3920135 21bd0e36-f7a9-4a5e-a9c7-6bc8d4e1587b /tidb/cdc/default/default/upstream/7364998688785556323 {"id":7364998688785556323,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 12:36:43 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1156111563.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714797404.3786836,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d68000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} {"level":"warn","ts":1714797404.3791697,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0021a48c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 Run finished, takes 2.296404042s INSERT - Takes(s): 2.3, Count: 5000, OPS: 2185.3, Avg(us): 4250, Min(us): 1421, Max(us): 67798, 95th(us): 7000, 99th(us): 51000 Sending interrupt signal to process Killing processes ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) script returned exit code 143 kill finished with exit code 0 table testSync.usertable exists table testSync.simple1 exists table testSync.simple2 exists script returned exit code 143 script returned exit code 143 ++ stop_tidb_cluster {"level":"warn","ts":1714797403.0075657,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00213dc00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} {"level":"warn","ts":1714797405.0074964,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00213dc00/127.0.0.1:2379","attempt":1,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} script returned exit code 143 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // withEnv [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // cache [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // node [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // dir [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // dir [Pipeline] // withCredentials [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // timeout [Pipeline] // stage [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] // container [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // container [Pipeline] // withEnv [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // node [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // podTemplate [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // withEnv [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G06' [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G00' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G02' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G04' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G05' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G07' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G12' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G13' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G14' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G15' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G16' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G17' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G18' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G19' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G20' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G21' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: 4eadb6d8-7377-442c-90b4-219ab79272ca Failed in branch Matrix - TEST_GROUP = 'G06' Finished: ABORTED