Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-ktt1p Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-ktt1p is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ktf96-ndd48 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e1646f4d3a0209ace1400a53b66b3ac4fbea2e43" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ktf96" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-ktt1p" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-ktt1p" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-ktt1p" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-ktt1p in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git rev-list --no-walk e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 5 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=45bfffba-3413-4f2a-99f5-a7427e14e429 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-ktt1p HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ktf96 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a1283f80f8ebe563bfe9f2bd09649a5cc54c1f9b857850fc6066be7891f3e273 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-ktt1p ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ktf96 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-ktt1p GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.0' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build1420528125=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-ktt1p bash' debug command: kubectl -n jenkins-tiflow exec -ti ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-ktt1p bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 15057 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 95b2cd9a62f74ce9 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 21 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Thu May 30 06:40:52 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11200/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11200/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11200/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11200/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/kv/shared_client.go, go.mod, go.sum, pkg/spanz/hash_map.go, pkg/util/seahash/LICENSE.md, pkg/util/seahash/seahash.go, pkg/util/seahash/seahash_test.go, scripts/check-copyright.sh] [Pipeline] echo diff file not matched: cdc/kv/shared_client.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-26b901c-8f92142) 208203776 bytes in 1.00 secs (208481579 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/.git/ .git HEAD is now at 8f92142d7 fix POST git-upload-pack (686 bytes) From https://github.com/pingcap/tiflow = [up to date] release-7.5 -> origin/release-7.5 = [up to date] refs/pull/11200/head -> origin/pr/11200/head Previous HEAD position was 8f92142d7 fix HEAD is now at 26b901cb5 dm: upgrade go-mysql library (#11043) (#11189) ๐Ÿšง Checkouting to base SHA:26b901cb5b23f8c42b05e5a00514b52bef2efe24... HEAD is now at 26b901cb5 dm: upgrade go-mysql library (#11043) (#11189) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: 26b901cb5b23f8c42b05e5a00514b52bef2efe24 26b901cb5 dm: upgrade go-mysql library (#11043) (#11189) bf93d7d63 dm: fix update dml loss if binary column is primary key (#10685) (#10691) 39a17af6e dep(*): move diff package to the tiflow and get rid of the tidb-tools (#11164) (#11168) ๐Ÿšง Pre-merge heads of pull requests to base SHA: 26b901cb5b23f8c42b05e5a00514b52bef2efe24 ... Updating 26b901cb5..8f92142d7 Fast-forward cdc/kv/shared_client.go | 2 +- go.mod | 10 --- go.sum | 34 --------- pkg/spanz/hash_map.go | 2 +- pkg/util/seahash/LICENSE.md | 9 +++ pkg/util/seahash/seahash.go | 159 +++++++++++++++++++++++++++++++++++++++ pkg/util/seahash/seahash_test.go | 75 ++++++++++++++++++ scripts/check-copyright.sh | 2 +- 8 files changed, 246 insertions(+), 47 deletions(-) create mode 100644 pkg/util/seahash/LICENSE.md create mode 100644 pkg/util/seahash/seahash.go create mode 100644 pkg/util/seahash/seahash_test.go ๐Ÿงพ Pre-merged result: 8f92142d7874bfad2a8db6ca100604c383fca9df 8f92142d7 fix 645f6fc14 This is an automated cherry-pick of #11193 26b901cb5 dm: upgrade go-mysql library (#11043) (#11189) โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-26b901c-8f92142 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh release-7.5 Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1575 0 --:--:-- --:--:-- --:--:-- 1640 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 174 0 --:--:-- --:--:-- --:--:-- 174 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 166 0 --:--:-- --:--:-- --:--:-- 179 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1910 0 --:--:-- --:--:-- --:--:-- 1952 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef/centos7/tidb-server.tar.gz 2024-05-30 14:41:27 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef/centos7/tidb-server.tar.gz [512424861/512424861] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/634e05a87ee0459846e48e8acea3f6bc5f6973a7/centos7/pd-server.tar.gz 2024-05-30 14:41:46 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/634e05a87ee0459846e48e8acea3f6bc5f6973a7/centos7/pd-server.tar.gz [173743994/173743994] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/1a2c88dcc22c62e1f138d440956214ce367e838b/centos7/tikv-server.tar.gz 2024-05-30 14:42:31 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/1a2c88dcc22c62e1f138d440956214ce367e838b/centos7/tikv-server.tar.gz [817702020/817702020] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-7.5/db0862135d4a94904069a37ec50c69cfe6099952/centos7/tiflash.tar.gz 2024-05-30 14:42:55 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-7.5/db0862135d4a94904069a37ec50c69cfe6099952/centos7/tiflash.tar.gz [263371625/263371625] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-30 14:42:59 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-30 14:43:00 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-30 14:43:01 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-30 14:43:02 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz 2024-05-30 14:43:04 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz [55896256/55896256] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-30 14:43:12 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.6G drwxr-sr-x. 6 jenkins jenkins 4.0K May 30 14:43 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 14:43 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 30 14:43 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 30 14:43 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 27 16:52 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 2023 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 27 16:52 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 2023 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 27 16:52 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 27 16:52 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 27 16:18 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 88M May 27 16:44 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 33M May 28 20:27 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 43M May 28 20:27 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M May 28 20:27 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 pd-recover -rwxr-xr-x. 1 jenkins jenkins 103M May 28 20:26 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M May 28 20:27 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M May 29 23:11 tidb-server -rwxr-xr-x. 1 jenkins jenkins 377M May 27 16:52 tiflash -rwxr-xr-x. 1 jenkins jenkins 357M May 28 18:52 tikv-server + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ./bin/ + ls -alh ./bin total 1.6G drwxr-sr-x. 6 jenkins jenkins 4.0K May 30 14:43 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 14:43 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 30 14:43 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 30 14:43 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 27 16:52 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 2023 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 27 16:52 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 2023 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 27 16:52 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 27 16:52 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 27 16:18 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 88M May 27 16:44 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 33M May 28 20:27 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 43M May 28 20:27 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M May 28 20:27 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 pd-recover -rwxr-xr-x. 1 jenkins jenkins 103M May 28 20:26 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M May 28 20:27 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M May 29 23:11 tidb-server -rwxr-xr-x. 1 jenkins jenkins 377M May 27 16:52 tiflash -rwxr-xr-x. 1 jenkins jenkins 357M May 28 18:52 tikv-server + ./bin/tidb-server -V Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 + ./bin/tikv-server -V TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.1.5-2-geae7b5d Git Commit Hash: eae7b5d149e30a02a04ec6116e08b0a8f07fe4aa Git Branch: disable-cgo-for-sync-diff UTC Build Time: 2024-05-06 08:40:40 Go Version: go1.22.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K May 30 14:43 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 14:43 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-58-g8f92142d7" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-30 06:43:16" -X "github.com/pingcap/tiflow/pkg/version.GitHash=8f92142d7874bfad2a8db6ca100604c383fca9df" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-58-g8f92142d7"' -o bin/cdc ./cmd/cdc/main.go go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240527072219-a97e6464c01d go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240522024016-df42997c2c57 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading golang.org/x/net v0.25.0 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240527072219-a97e6464c01d go: downloading github.com/tikv/pd/client v0.0.0-20240210135946-3488a653ddd9 go: downloading github.com/prometheus/client_golang v1.19.1 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240510070134-e7f530c64790 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/r3labs/diff v1.1.0 go: downloading golang.org/x/sys v0.20.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240507075657-2bd4573edde2 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.34.1 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.15.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading golang.org/x/text v0.15.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.21.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.6 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20240507090649-2bf6bb0cb996 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.20.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/sasha-s/go-deadlock v0.3.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-58-g8f92142d7" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-30 06:47:10" -X "github.com/pingcap/tiflow/pkg/version.GitHash=8f92142d7874bfad2a8db6ca100604c383fca9df" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-58-g8f92142d7"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-58-g8f92142d7" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-30 06:47:18" -X "github.com/pingcap/tiflow/pkg/version.GitHash=8f92142d7874bfad2a8db6ca100604c383fca9df" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-58-g8f92142d7"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-58-g8f92142d7" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-30 06:47:26" -X "github.com/pingcap/tiflow/pkg/version.GitHash=8f92142d7874bfad2a8db6ca100604c383fca9df" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-58-g8f92142d7"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-58-g8f92142d7" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-30 06:47:26" -X "github.com/pingcap/tiflow/pkg/version.GitHash=8f92142d7874bfad2a8db6ca100604c383fca9df" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-58-g8f92142d7"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-58-g8f92142d7" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-30 06:47:26" -X "github.com/pingcap/tiflow/pkg/version.GitHash=8f92142d7874bfad2a8db6ca100604c383fca9df" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-58-g8f92142d7"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/DataDog/zstd v1.5.5 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading go.uber.org/dig v1.13.0 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading modernc.org/sqlite v1.29.8 go: downloading modernc.org/libc v1.50.4 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading modernc.org/memory v1.8.0 go: downloading modernc.org/mathutil v1.6.0 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-58-g8f92142d7" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-30 06:47:26" -X "github.com/pingcap/tiflow/pkg/version.GitHash=8f92142d7874bfad2a8db6ca100604c383fca9df" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-58-g8f92142d7"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-58-g8f92142d7" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-30 06:47:26" -X "github.com/pingcap/tiflow/pkg/version.GitHash=8f92142d7874bfad2a8db6ca100604c383fca9df" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-58-g8f92142d7"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.1G drwxr-sr-x. 2 jenkins jenkins 4.0K May 30 14:53 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 14:43 .. -rwxr-xr-x. 1 jenkins jenkins 208M May 30 14:53 cdc -rwxr-xr-x. 1 jenkins jenkins 336M May 30 14:52 cdc.test -rwxr-xr-x. 1 jenkins jenkins 178M May 30 14:47 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 178M May 30 14:47 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 177M May 30 14:47 cdc_storage_consumer + ./bin/cdc version Release Version: v7.5.1-58-g8f92142d7 Git Commit Hash: 8f92142d7874bfad2a8db6ca100604c383fca9df Git Branch: HEAD UTC Build Time: 2024-05-30 06:47:26 Go Version: go version go1.21.0 linux/amd64 Failpoint Build: true [Pipeline] } Cache not saved (binary/pingcap/tiflow/cdc-integration-test/rev-26b901c-8f92142 already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ./bin/ + ls -alh ./bin total 2.6G drwxr-sr-x. 6 jenkins jenkins 4.0K May 30 14:53 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 14:43 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 30 14:53 bin -rwxr-xr-x. 1 jenkins jenkins 208M May 30 14:53 cdc -rwxr-xr-x. 1 jenkins jenkins 336M May 30 14:52 cdc.test -rwxr-xr-x. 1 jenkins jenkins 178M May 30 14:47 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 178M May 30 14:47 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 177M May 30 14:47 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 14:53 etc -rwxr-xr-x. 1 jenkins jenkins 17M May 30 14:53 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 30 14:53 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 30 14:53 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 14:53 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 30 14:53 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 30 14:53 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 30 14:53 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 30 14:53 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 30 14:53 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 30 14:53 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 30 14:53 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 88M May 30 14:53 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 30 14:53 minio -rwxr-xr-x. 1 jenkins jenkins 33M May 30 14:53 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 43M May 30 14:53 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M May 30 14:53 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 30 14:53 pd-recover -rwxr-xr-x. 1 jenkins jenkins 103M May 30 14:53 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M May 30 14:53 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 30 14:53 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 14:53 share -rwxr-xr-x. 1 jenkins jenkins 29M May 30 14:53 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 30 14:53 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M May 30 14:53 tidb-server -rwxr-xr-x. 1 jenkins jenkins 377M May 30 14:53 tiflash -rwxr-xr-x. 1 jenkins jenkins 357M May 30 14:53 tikv-server [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 132.76 secs (24908764 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7 [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4 [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-dthvf-kkzlm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d9b7577413855b52b72f21e2bdcfa00ef92e3a5e" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-dthvf" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test [Pipeline] podTemplate [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t [Pipeline] podTemplate Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-cr7wk-shgzq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "411ac7642f8d5e532393a99fd7e15798c1915642" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-cr7wk" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" [Pipeline] { Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] node [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1db68a4b; decorates RemoteLauncher[hudson.remoting.Channel@2d5a78a:JNLP4-connect connection from 10.233.72.231/10.233.72.231:44140] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-nmb95-jz54w --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ecb165e8ffee45a5221a8a3ae0051903c7183f39" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-nmb95" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test [Pipeline] { Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3c1fe304; decorates RemoteLauncher[hudson.remoting.Channel@14a98852:JNLP4-connect connection from 10.233.69.113/10.233.69.113:55956] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6f84c955; decorates RemoteLauncher[hudson.remoting.Channel@70b6d8bd:JNLP4-connect connection from 10.233.69.2/10.233.69.2:57726] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-8nqlr-5fltp --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2f58178942a143a4c08e3b60845d3bf3a5bf95c6" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-8nqlr" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate The recommended git tool is: git [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-g14fn-8twq3 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2afad8c000912bef39a0282fc24721e852111022" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-g14fn" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test [Pipeline] { Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) [Pipeline] checkout The recommended git tool is: git Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-7bzbg-nnq6v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2453be1d2b80fc62e8d5fc2389e81e48b79ee183" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-7bzbg" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2e03fe8f; decorates RemoteLauncher[hudson.remoting.Channel@3d29cee:JNLP4-connect connection from 10.233.69.160/10.233.69.160:51802] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@d5536b5; decorates RemoteLauncher[hudson.remoting.Channel@6517bcf5:JNLP4-connect connection from 10.233.86.124/10.233.86.124:58260] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-bjphl-3bs6m --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2f7d16bb9f426735446d7d75f015bfb22198970d" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-bjphl" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-pvszg-6dhnd --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b0a4bc13f46453a1520b54fce77db0a72716f2b4" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-pvszg" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-94c5f-8j6fg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "eef5abfde44af5a32909e391aa67384eb37daaa1" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-94c5f" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-vbl0n-7w55r --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "bd75a50a8594dc656062fe66119ec2bf84456d3a" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-vbl0n" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-whfsd-prz0g --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4df39138575800fb928e8785e2acabedcf66b961" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-whfsd" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Still waiting to schedule task โ€˜ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wrโ€™ is offline Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 6.61 secs (500183346 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] container [Pipeline] { The recommended git tool is: git [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3 The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-j67b2 Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-mjlz0 The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] } [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // timeout [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@24cdedd8; decorates RemoteLauncher[hudson.remoting.Channel@7006deb4:JNLP4-connect connection from 10.233.127.79/10.233.127.79:60492] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] // container [Pipeline] sh Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@56968026; decorates RemoteLauncher[hudson.remoting.Channel@3cf77b0d:JNLP4-connect connection from 10.233.108.155/10.233.108.155:59318] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@20137c89; decorates RemoteLauncher[hudson.remoting.Channel@512f99ad:JNLP4-connect connection from 10.233.66.71/10.233.66.71:48182] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5404ca50; decorates RemoteLauncher[hudson.remoting.Channel@18f6dc25:JNLP4-connect connection from 10.233.69.23/10.233.69.23:44908] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_txn_v2 kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=8d624702-efb8-42ca-a677-90c1026806c0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-dthvf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-dthvf ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: kafka... <<================= No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1f4f3105; decorates RemoteLauncher[hudson.remoting.Channel@7d67e2e0:JNLP4-connect connection from 10.233.72.185/10.233.72.185:49326] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3f279d73; decorates RemoteLauncher[hudson.remoting.Channel@6791a67b:JNLP4-connect connection from 10.233.97.58/10.233.97.58:37196] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Thu May 30 14:56:02 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-pz6n0-qnsqr --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9be8060274a16b82c107eb8e8323414b54626218" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-pz6n0" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-3v2gz-pccmj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "988dbaac5c3a70f7fceb0cdec176490d074de2bc" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-3v2gz" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-nhz9r-q8fr9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d2295d5cb49f97da10796e2c109d3dbdfd95a72e" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-nhz9r" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-j67b2 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-2gdb1-33pqq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8c879634ca026ed06db402ca4f8e683069e9afd9" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-2gdb1" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-j67b2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-j67b2" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-j67b2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-j67b2 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D431D960177F06 < X-Xss-Protection: 1; mode=block < Date: Thu, 30 May 2024 06:56:07 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Thu May 30 14:56:07 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: kafka... <<================= [Thu May 30 14:56:10 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ld79s-322fs --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "57b2adc4723ea412b0f628926a6d4350f671749c" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ld79s" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ls1gk-bmwbl Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-mjlz0 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-5rms5-chhbt --- apiVersion: "v1" kind: "Pod" metadata: annotations: --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c7e5bfd2018b92c9d63392c4171cb9b8f1b24f9b" jenkins/label-digest: "15e5c43ff899e98db75e33e1b6a47b6865d1e507" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-5rms5" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-mjlz0" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ls1gk" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" resources: limits: cpu: "2000m" memory: "4Gi" requests: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" name: "workspace-volume" readOnly: false - args: - "cat" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" name: "kafka" resources: limits: cpu: "2000m" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "report" resources: limits: memory: "256Mi" cpu: "100m" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" name: "report" resources: limits: memory: "256Mi" cpu: "100m" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-mjlz0" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-mjlz0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: - emptyDir: medium: "" name: "workspace-volume" medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-mjlz0 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: kafka... <<================= [Thu May 30 14:56:13 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: kafka... <<================= [Thu May 30 14:56:16 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 14.25 secs (232053655 bytes/sec) [Pipeline] { [Pipeline] cache find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D431DCAF703A7B < X-Xss-Protection: 1; mode=block < Date: Thu, 30 May 2024 06:56:21 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Thu May 30 14:56:21 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: kafka... <<================= [Thu May 30 14:56:25 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_txn_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/kafka_big_txn_v2 Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21257d240013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:1869, start at 2024-05-30 14:56:42.865493926 +0800 CST m=+5.661198676 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:58:42.872 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:56:42.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:46:42.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21257d240013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:1869, start at 2024-05-30 14:56:42.865493926 +0800 CST m=+5.661198676 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:58:42.872 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:56:42.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:46:42.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21257ea80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:1953, start at 2024-05-30 14:56:42.953516298 +0800 CST m=+5.590375825 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:58:42.961 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:56:42.922 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:46:42.922 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Thu May 30 14:56:47 CST 2024] <<<<<< START cdc server in kafka_big_txn_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.32353237.out server --log-file /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:56:50 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 {"id":"fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8493353bc fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 /tidb/cdc/default/default/upstream/7374683017226072260 {"id":7374683017226072260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 {"id":"fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8493353bc fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 /tidb/cdc/default/default/upstream/7374683017226072260 {"id":7374683017226072260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 {"id":"fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8493353bc fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 /tidb/cdc/default/default/upstream/7374683017226072260 {"id":7374683017226072260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.cli.3278.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450114934833152002 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114934833152002 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 14:56:52 CST 2024] <<<<<< START cdc server in kafka_big_txn_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.33023304.out server --log-file /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:56:52 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 {"id":"fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8493353bc fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 /tidb/cdc/default/default/upstream/7374683017226072260 {"id":7374683017226072260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 {"id":"fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8493353bc fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 /tidb/cdc/default/default/upstream/7374683017226072260 {"id":7374683017226072260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 {"id":"fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8493353bc fef4a5d5-1187-40a8-9ee3-07cdc7a0fc32 /tidb/cdc/default/default/upstream/7374683017226072260 {"id":7374683017226072260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.cli.3331.out cli changefeed create --start-ts=450114934833152002 '--sink-uri=kafka://127.0.0.1:9092/ticdc-big-txn-test-32125?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_txn_v2/conf/changefeed.toml Create changefeed successfully! ID: c73e23c3-7391-438f-b6d0-0faeaaf17c6a Info: {"upstream_id":7374683017226072260,"namespace":"default","id":"c73e23c3-7391-438f-b6d0-0faeaaf17c6a","sink_uri":"kafka://127.0.0.1:9092/ticdc-big-txn-test-32125?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T14:56:53.105515199+08:00","start_ts":450114934833152002,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114934833152002,"checkpoint_ts":450114934833152002,"checkpoint_time":"2024-05-30 14:56:51.125"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ***************** properties ***************** "readproportion"="0" "readallfields"="true" "mysql.port"="4000" "threadcount"="1" "recordcount"="5000" "mysql.user"="root" "fieldcount"="100" "updateproportion"="0" "requestdistribution"="uniform" "workload"="core" "insertproportion"="0" "operationcount"="0" "mysql.db"="big_txn" "mysql.host"="127.0.0.1" "dotransactions"="false" "scanproportion"="0" ********************************************** Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 36.95 secs (89489395 bytes/sec) [Pipeline] { [Pipeline] cache INSERT - Takes(s): 10.0, Count: 2616, OPS: 261.8, Avg(us): 3459, Min(us): 1292, Max(us): 96121, 95th(us): 3000, 99th(us): 82000 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 7.99 secs (414036855 bytes/sec) [Pipeline] { [Pipeline] cache Run finished, takes 18.934589386s INSERT - Takes(s): 18.9, Count: 5000, OPS: 264.2, Avg(us): 3479, Min(us): 1292, Max(us): 114891, 95th(us): 3000, 99th(us): 81000 [Thu May 30 14:57:14 CST 2024] <<<<<< START kafka consumer in kafka_big_txn_v2 case >>>>>> table big_txn.usertable not exists for 1-th check, retry later table big_txn.usertable not exists for 2-th check, retry later table big_txn.usertable exists Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 6.21 secs (532454435 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + awk {$1=$1;print} + + grep brokers nc localhost 2181 + grep -F -w /brokers/ids/1 + echo dump /brokers/ids/1 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] } [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] stage The recommended git tool is: git [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // container No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@dcedace; decorates RemoteLauncher[hudson.remoting.Channel@66b9f4e4:JNLP4-connect connection from 10.233.90.27/10.233.90.27:45154] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] // container No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@53767a03; decorates RemoteLauncher[hudson.remoting.Channel@4e747cb4:JNLP4-connect connection from 10.233.90.4/10.233.90.4:44176] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] // container [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7a7bad7; decorates RemoteLauncher[hudson.remoting.Channel@6e1c8b2b:JNLP4-connect connection from 10.233.88.15/10.233.88.15:52422] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@33e15a5b; decorates RemoteLauncher[hudson.remoting.Channel@14ee7a79:JNLP4-connect connection from 10.233.69.217/10.233.69.217:57222] will be ignored (a typical symptom is the Git executable not being run inside a designated container) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2852fc43; decorates RemoteLauncher[hudson.remoting.Channel@32390814:JNLP4-connect connection from 10.233.72.15/10.233.72.15:39776] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@491d832d; decorates RemoteLauncher[hudson.remoting.Channel@29208f67:JNLP4-connect connection from 10.233.88.200/10.233.88.200:53078] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3d81d57f; decorates RemoteLauncher[hudson.remoting.Channel@7b50e0c:JNLP4-connect connection from 10.233.88.72/10.233.88.72:35084] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility changefeed_dup_error_restart kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro lossy_ddl storage_csv_update PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=566a6f98-a214-43a9-8e90-86285bf7e075 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-cr7wk GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-cr7wk ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env [Pipeline] sh find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: kafka... <<================= [Thu May 30 14:57:25 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G01 Run cases: http_api http_api_tls api_v2 kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=c309eddb-7a85-415f-a00f-b54d5f280755 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-nmb95 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-nmb95 ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: kafka... <<================= Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] // container [Pipeline] sh [Thu May 30 14:57:25 CST 2024] <<<<<< run test case http_api success! >>>>>> + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G03 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=e34dac38-1758-48dc-808e-d6baa66c7a86 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-8nqlr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-8nqlr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=14ff5bb3-7a3a-4755-8a39-c70cf511d4bd BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-g14fn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-g14fn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" [Pipeline] { [Pipeline] cache Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: kafka... <<================= [Thu May 30 14:57:28 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: kafka... <<================= [Thu May 30 14:57:28 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: kafka... <<================= kafka downstream isn't support syncpoint record [Thu May 30 14:57:31 CST 2024] <<<<<< run test case syncpoint success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: kafka... <<================= Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: kafka... <<================= [Thu May 30 14:57:34 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 7.24 secs (456431801 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: kafka... <<================= [Thu May 30 14:57:37 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_basic Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: kafka... <<================= [Thu May 30 14:57:40 CST 2024] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212915e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:1126, start at 2024-05-30 14:57:41.799047216 +0800 CST m=+6.489432463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:41.808 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:41.801 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:41.801 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212915e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:1126, start at 2024-05-30 14:57:41.799047216 +0800 CST m=+6.489432463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:41.808 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:41.801 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:41.801 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212918e00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:1189, start at 2024-05-30 14:57:41.992055068 +0800 CST m=+6.520610714 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:42.001 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:41.994 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:41.994 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 6.19 secs (533997783 bytes/sec) [Pipeline] { [Pipeline] cache + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2460.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... + set +x + tso='450114949310578689 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114949310578689 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 14:57:47 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.24862488.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21297818000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:1299, start at 2024-05-30 14:57:48.050985881 +0800 CST m=+6.449604011 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:48.059 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:48.038 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:48.038 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21297818000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:1299, start at 2024-05-30 14:57:48.050985881 +0800 CST m=+6.449604011 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:48.059 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:48.038 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:48.038 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21297b5c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:1368, start at 2024-05-30 14:57:48.297143373 +0800 CST m=+6.593535571 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:48.307 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:48.297 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:48.297 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212973440009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:1184, start at 2024-05-30 14:57:47.738549571 +0800 CST m=+5.825781855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:47.746 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:47.729 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:47.729 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212973440009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:1184, start at 2024-05-30 14:57:47.738549571 +0800 CST m=+5.825781855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:47.746 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:47.729 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:47.729 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2129741c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:1255, start at 2024-05-30 14:57:47.796862006 +0800 CST m=+5.770172248 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:47.804 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:47.783 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:47.783 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:57:50 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a26594c-be0c-46c9-ad35-eb178fda9e7b {"id":"1a26594c-be0c-46c9-ad35-eb178fda9e7b","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a1469c6 1a26594c-be0c-46c9-ad35-eb178fda9e7b /tidb/cdc/default/default/upstream/7374683269195753215 {"id":7374683269195753215,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a26594c-be0c-46c9-ad35-eb178fda9e7b {"id":"1a26594c-be0c-46c9-ad35-eb178fda9e7b","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a1469c6 1a26594c-be0c-46c9-ad35-eb178fda9e7b /tidb/cdc/default/default/upstream/7374683269195753215 {"id":7374683269195753215,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a26594c-be0c-46c9-ad35-eb178fda9e7b {"id":"1a26594c-be0c-46c9-ad35-eb178fda9e7b","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a1469c6 1a26594c-be0c-46c9-ad35-eb178fda9e7b /tidb/cdc/default/default/upstream/7374683269195753215 {"id":7374683269195753215,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2529.out cli changefeed create --start-ts=450114949310578689 '--sink-uri=kafka://127.0.0.1:9092/ticdc-row-format-test-655?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: d56239d6-8e12-4a81-8af2-e9644bfd33b8 Info: {"upstream_id":7374683269195753215,"namespace":"default","id":"d56239d6-8e12-4a81-8af2-e9644bfd33b8","sink_uri":"kafka://127.0.0.1:9092/ticdc-row-format-test-655?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T14:57:51.476936748+08:00","start_ts":450114949310578689,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114949310578689,"checkpoint_ts":450114949310578689,"checkpoint_time":"2024-05-30 14:57:46.352"} PASS Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Thu May 30 14:57:52 CST 2024] <<<<<< START cdc server in kafka_simple_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.27242726.out server --log-file /tmp/tidb_cdc_test/kafka_simple_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_basic/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2466.out cli tso query --pd=http://127.0.0.1:2379 + set +x [Thu May 30 14:57:52 CST 2024] <<<<<< START kafka consumer in row_format case >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450114950995116033 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114950995116033 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:57:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81a4daca-fd47-49e8-8196-0c14e34f80e7 {"id":"81a4daca-fd47-49e8-8196-0c14e34f80e7","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a36bdbc 81a4daca-fd47-49e8-8196-0c14e34f80e7 /tidb/cdc/default/default/upstream/7374683297765532736 {"id":7374683297765532736,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81a4daca-fd47-49e8-8196-0c14e34f80e7 {"id":"81a4daca-fd47-49e8-8196-0c14e34f80e7","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a36bdbc 81a4daca-fd47-49e8-8196-0c14e34f80e7 /tidb/cdc/default/default/upstream/7374683297765532736 {"id":7374683297765532736,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81a4daca-fd47-49e8-8196-0c14e34f80e7 {"id":"81a4daca-fd47-49e8-8196-0c14e34f80e7","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a36bdbc 81a4daca-fd47-49e8-8196-0c14e34f80e7 /tidb/cdc/default/default/upstream/7374683297765532736 {"id":7374683297765532736,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.2769.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-simple-basic-27823?protocol=simple' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic/conf/changefeed.toml -c simple-basic ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: simple-basic Info: {"upstream_id":7374683297765532736,"namespace":"default","id":"simple-basic","sink_uri":"kafka://127.0.0.1:9092/ticdc-simple-basic-27823?protocol=simple","create_time":"2024-05-30T14:57:55.801004188+08:00","start_ts":450114951705526277,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":5,"send_bootstrap_in_msg_count":100,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114951705526277,"checkpoint_ts":450114951705526277,"checkpoint_time":"2024-05-30 14:57:55.488"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.db"="sink_retry" "updateproportion"="0" "mysql.user"="root" "dotransactions"="false" "readallfields"="true" "scanproportion"="0" "workload"="core" "operationcount"="0" "recordcount"="10" "requestdistribution"="uniform" "threadcount"="2" "insertproportion"="0" ********************************************** Run finished, takes 9.13771ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2139.2, Avg(us): 1730, Min(us): 954, Max(us): 4324, 95th(us): 5000, 99th(us): 5000 [Thu May 30 14:57:56 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.25102512.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + set +x + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:58:01 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2cfb61b-8298-47e7-9743-89f5c6ff31b2 {"id":"e2cfb61b-8298-47e7-9743-89f5c6ff31b2","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a2e1cf1 e2cfb61b-8298-47e7-9743-89f5c6ff31b2 /tidb/cdc/default/default/upstream/7374683293718485916 {"id":7374683293718485916,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2cfb61b-8298-47e7-9743-89f5c6ff31b2 {"id":"e2cfb61b-8298-47e7-9743-89f5c6ff31b2","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a2e1cf1 e2cfb61b-8298-47e7-9743-89f5c6ff31b2 /tidb/cdc/default/default/upstream/7374683293718485916 {"id":7374683293718485916,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2cfb61b-8298-47e7-9743-89f5c6ff31b2 {"id":"e2cfb61b-8298-47e7-9743-89f5c6ff31b2","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a2e1cf1 e2cfb61b-8298-47e7-9743-89f5c6ff31b2 /tidb/cdc/default/default/upstream/7374683293718485916 {"id":7374683293718485916,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2551.out cli changefeed create --start-ts=450114950995116033 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-14917?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212a2ce8000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:1639, start at 2024-05-30 14:57:59.626174971 +0800 CST m=+6.343959710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:59.635 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:59.610 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:59.610 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212a2ce8000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:1639, start at 2024-05-30 14:57:59.626174971 +0800 CST m=+6.343959710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:59.635 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:59.610 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:59.610 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212a2e980014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:1710, start at 2024-05-30 14:57:59.759191197 +0800 CST m=+6.366545620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-14:59:59.777 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:57:59.767 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:47:59.767 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Create changefeed successfully! ID: 3380f8db-3138-4229-9514-5504fd976f65 Info: {"upstream_id":7374683293718485916,"namespace":"default","id":"3380f8db-3138-4229-9514-5504fd976f65","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-14917?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T14:58:01.86029503+08:00","start_ts":450114950995116033,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114950995116033,"checkpoint_ts":450114950995116033,"checkpoint_time":"2024-05-30 14:57:52.778"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Thu May 30 14:58:02 CST 2024] <<<<<< START kafka consumer in kafka_simple_basic case >>>>>> table test.finish_mark_for_ddl not exists for 1-th check, retry later [Thu May 30 14:58:03 CST 2024] <<<<<< START cdc server in kafka_big_messages case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages.29632965.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark_for_ddl not exists for 2-th check, retry later + set +x [Thu May 30 14:58:06 CST 2024] <<<<<< START kafka consumer in sink_retry case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:58:06 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7645a06c-1b31-47db-89ad-dc20cd3627ab {"id":"7645a06c-1b31-47db-89ad-dc20cd3627ab","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a5a92c4 7645a06c-1b31-47db-89ad-dc20cd3627ab /tidb/cdc/default/default/upstream/7374683347274645985 {"id":7374683347274645985,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7645a06c-1b31-47db-89ad-dc20cd3627ab {"id":"7645a06c-1b31-47db-89ad-dc20cd3627ab","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a5a92c4 7645a06c-1b31-47db-89ad-dc20cd3627ab /tidb/cdc/default/default/upstream/7374683347274645985 {"id":7374683347274645985,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7645a06c-1b31-47db-89ad-dc20cd3627ab {"id":"7645a06c-1b31-47db-89ad-dc20cd3627ab","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84a5a92c4 7645a06c-1b31-47db-89ad-dc20cd3627ab /tidb/cdc/default/default/upstream/7374683347274645985 {"id":7374683347274645985,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 7d10da12-a09b-4bf0-8b8d-0aea1c4160c0 Info: {"upstream_id":7374683347274645985,"namespace":"default","id":"7d10da12-a09b-4bf0-8b8d-0aea1c4160c0","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-05-30T14:58:07.156022573+08:00","start_ts":450114953873719297,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114953873719297,"checkpoint_ts":450114953873719297,"checkpoint_time":"2024-05-30 14:58:03.759"} [Thu May 30 14:58:07 CST 2024] <<<<<< START kafka consumer in kafka_big_messages case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/atomic v1.11.0 table test.finish_mark_for_ddl not exists for 3-th check, retry later table row_format.finish_mark not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 19.65 secs (168310043 bytes/sec) [Pipeline] { [Pipeline] cache table test.finish_mark_for_ddl not exists for 4-th check, retry later table row_format.finish_mark not exists for 2-th check, retry later table test.finish_mark_for_ddl exists + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.2871.out cli changefeed pause -c simple-basic PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... table row_format.finish_mark not exists for 3-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.2901.out cli changefeed resume -c simple-basic PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table row_format.finish_mark not exists for 4-th check, retry later + set +x table kafka_big_messages.test exists check diff failed 1-th time, retry later table row_format.finish_mark not exists for 5-th check, retry later table row_format.finish_mark not exists for 6-th check, retry later check diff failed 2-th time, retry later table test.finish_mark not exists for 1-th check, retry later table row_format.finish_mark not exists for 7-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 14:58:21 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> table row_format.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 14:58:24 CST 2024] <<<<<< run test case row_format success! >>>>>> table test.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 5-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 17.21 secs (192154925 bytes/sec) [Pipeline] { [Pipeline] cache table test.finish_mark not exists for 6-th check, retry later table test.finish_mark not exists for 7-th check, retry later table test.finish_mark not exists for 8-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 9-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/kafka_compression Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table test.finish_mark not exists for 10-th check, retry later table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "operationcount"="0" "mysql.port"="4000" "dotransactions"="false" "workload"="core" "requestdistribution"="uniform" "updateproportion"="0" "insertproportion"="0" "scanproportion"="0" "mysql.db"="sink_retry" "threadcount"="2" "readproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "mysql.user"="root" "recordcount"="10" ********************************************** Run finished, takes 5.644057ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2481.0, Avg(us): 950, Min(us): 609, Max(us): 2074, 95th(us): 3000, 99th(us): 3000 start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Verifying downstream PD is started... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table test.finish_mark not exists for 11-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.finish_mark not exists for 12-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.finish_mark not exists for 13-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 12.12 secs (272869400 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212d1a540013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:5171, start at 2024-05-30 14:58:47.616088822 +0800 CST m=+5.980819416 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:00:47.623 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:58:47.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:48:47.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212d1a540013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:5171, start at 2024-05-30 14:58:47.616088822 +0800 CST m=+5.980819416 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:00:47.623 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:58:47.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:48:47.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212d1ebc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:5246, start at 2024-05-30 14:58:47.860890105 +0800 CST m=+6.071819118 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:00:47.869 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:58:47.855 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:48:47.855 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212d3d3c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:3680, start at 2024-05-30 14:58:49.809530303 +0800 CST m=+6.099467491 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:00:49.816 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:58:49.807 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:48:49.807 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 2-th check, retry later [Thu May 30 14:58:52 CST 2024] <<<<<< START cdc server in kafka_compression case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.64606462.out server --log-file /tmp/tidb_cdc_test/kafka_compression/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_compression/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212d3d3c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:3680, start at 2024-05-30 14:58:49.809530303 +0800 CST m=+6.099467491 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:00:49.816 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:58:49.807 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:48:49.807 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212d3df80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:3748, start at 2024-05-30 14:58:49.888207286 +0800 CST m=+6.085614368 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:00:49.897 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:58:49.904 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:48:49.904 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 3-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.4989.out cli tso query --pd=http://127.0.0.1:2379 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 8.97 secs (368794869 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { table test.finish_mark not exists for 4-th check, retry later [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:58:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0322f371-b6b1-47af-a0f0-8e927d21ba86 {"id":"0322f371-b6b1-47af-a0f0-8e927d21ba86","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b1acfc2 0322f371-b6b1-47af-a0f0-8e927d21ba86 /tidb/cdc/default/default/upstream/7374683547363434342 {"id":7374683547363434342,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0322f371-b6b1-47af-a0f0-8e927d21ba86 {"id":"0322f371-b6b1-47af-a0f0-8e927d21ba86","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b1acfc2 0322f371-b6b1-47af-a0f0-8e927d21ba86 /tidb/cdc/default/default/upstream/7374683547363434342 {"id":7374683547363434342,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0322f371-b6b1-47af-a0f0-8e927d21ba86 {"id":"0322f371-b6b1-47af-a0f0-8e927d21ba86","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b1acfc2 0322f371-b6b1-47af-a0f0-8e927d21ba86 /tidb/cdc/default/default/upstream/7374683547363434342 {"id":7374683547363434342,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6511.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + set +x + tso='450114967177003009 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114967177003009 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 14:58:55 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.50145016.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + set +x + tso='450114967456710658 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114967456710658 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6550.out cli changefeed create --start-ts=450114967456710658 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip' -c gzip + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 Create changefeed successfully! ID: gzip Info: {"upstream_id":7374683547363434342,"namespace":"default","id":"gzip","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=gzip","create_time":"2024-05-30T14:58:57.479012056+08:00","start_ts":450114967456710658,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114967456710658,"checkpoint_ts":450114967456710658,"checkpoint_time":"2024-05-30 14:58:55.574"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 table test.finish_mark not exists for 5-th check, retry later + set +x [Thu May 30 14:58:58 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/30 14:58:57.422 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/30 14:58:57.472 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/30 14:58:57.589 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/30 14:58:57.600 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/30 14:58:58.555 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/30 14:58:58.594 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]: syntax error: operand expected (error token is "[2024/05/30 14:58:57.422 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/30 14:58:57.472 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/30 14:58:57.589 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/30 14:58:57.600 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/30 14:58:58.555 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/30 14:58:58.594 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]") table test.gzip_finish_mark not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:58:59 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/925eddd2-2ad2-4748-9dff-aa5bf86211a2 {"id":"925eddd2-2ad2-4748-9dff-aa5bf86211a2","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b238ac3 925eddd2-2ad2-4748-9dff-aa5bf86211a2 /tidb/cdc/default/default/upstream/7374683563584260401 {"id":7374683563584260401,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/925eddd2-2ad2-4748-9dff-aa5bf86211a2 {"id":"925eddd2-2ad2-4748-9dff-aa5bf86211a2","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b238ac3 925eddd2-2ad2-4748-9dff-aa5bf86211a2 /tidb/cdc/default/default/upstream/7374683563584260401 {"id":7374683563584260401,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/925eddd2-2ad2-4748-9dff-aa5bf86211a2 {"id":"925eddd2-2ad2-4748-9dff-aa5bf86211a2","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b238ac3 925eddd2-2ad2-4748-9dff-aa5bf86211a2 /tidb/cdc/default/default/upstream/7374683563584260401 {"id":7374683563584260401,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5063.out cli changefeed create --start-ts=450114967177003009 '--sink-uri=kafka://127.0.0.1:9092/ticdc-drop-tables-test-29245?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' table test.finish_mark not exists for 6-th check, retry later Create changefeed successfully! ID: 9aed6ce2-c19e-4057-bbc5-d111b282df7d Info: {"upstream_id":7374683563584260401,"namespace":"default","id":"9aed6ce2-c19e-4057-bbc5-d111b282df7d","sink_uri":"kafka://127.0.0.1:9092/ticdc-drop-tables-test-29245?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T14:58:59.610934941+08:00","start_ts":450114967177003009,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114967177003009,"checkpoint_ts":450114967177003009,"checkpoint_time":"2024-05-30 14:58:54.507"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.gzip_finish_mark not exists for 2-th check, retry later + set +x [Thu May 30 14:59:01 CST 2024] <<<<<< START kafka consumer in drop_many_tables case >>>>>> table test.finish_mark not exists for 7-th check, retry later [Pipeline] withEnv [Pipeline] { table drop_tables.c not exists for 1-th check, retry later [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) table test.finish_mark not exists for 8-th check, retry later table test.gzip_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6674.out cli changefeed pause -c gzip PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min table drop_tables.c not exists for 2-th check, retry later [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] // container [Pipeline] // container [Pipeline] // container + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6700.out cli changefeed remove -c gzip [Pipeline] // container [Pipeline] // container [Pipeline] sh table test.finish_mark not exists for 9-th check, retry later [Pipeline] sh [Pipeline] sh Changefeed remove successfully. ID: gzip CheckpointTs: 450114968360845353 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=f2b6e7d5-b8f2-46fe-83cc-27a9cd0b1cc7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-bjphl GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-bjphl ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: kafka... <<================= + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G08 [Pipeline] sh The 1 times to try to start tidb cluster... Run cases: processor_err_chan changefeed_reconstruct multi_capture PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=80d913d1-77b9-4f97-b44e-afd5983c9151 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-7bzbg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-7bzbg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G09 Run cases: gc_safepoint changefeed_pause_resume cli savepoint synced_status PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=d7c4bb23-cf93-4ed5-b8aa-dd66ffe2c04d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-pvszg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787 ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-pvszg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: kafka... <<================= table drop_tables.c not exists for 3-th check, retry later [Pipeline] sh The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=85579780-a494-44cb-8ef5-5104aceac8a3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-94c5f GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-94c5f GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: kafka... <<================= [Pipeline] // container [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=7ccb73d6-0e34-4c3a-ae2a-4ba26f467304 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-vbl0n GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-vbl0n ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: kafka... <<================= table test.finish_mark exists The 1 times to try to start tidb cluster... + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6731.out cli tso query --pd=http://127.0.0.1:2379 check diff successfully [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache wait process cdc.test exit for 1-th time... table drop_tables.c not exists for 4-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... + set +x + tso='450114970615283715 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114970615283715 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6757.out cli changefeed create --start-ts=450114970615283715 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy' -c snappy Create changefeed successfully! ID: snappy Info: {"upstream_id":7374683547363434342,"namespace":"default","id":"snappy","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=snappy","create_time":"2024-05-30T14:59:09.518864804+08:00","start_ts":450114970615283715,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114970615283715,"checkpoint_ts":450114970615283715,"checkpoint_time":"2024-05-30 14:59:07.623"} PASS cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 14:59:09 CST 2024] <<<<<< run test case kafka_simple_basic success! >>>>>> coverage: 2.4% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... The 1 times to try to start tidb cluster... table sink_retry.finish_mark_2 exists check diff successfully table drop_tables.c not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... + set +x [Thu May 30 14:59:10 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/30 14:59:09.475 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/30 14:59:09.513 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/30 14:59:09.615 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/30 14:59:09.626 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/30 14:59:10.607 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/30 14:59:10.617 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]: syntax error: operand expected (error token is "[2024/05/30 14:59:09.475 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/30 14:59:09.513 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/30 14:59:09.615 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/30 14:59:09.626 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/30 14:59:10.607 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/30 14:59:10.617 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]") table test.snappy_finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... Verifying downstream PD is started... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G10 Run cases: default_value simple cdc_server_tips event_filter PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=d249faa0-1249-4a4a-8e01-c8d2f4fa483b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-whfsd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-whfsd ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: kafka... <<================= cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 14:59:12 CST 2024] <<<<<< run test case sink_retry success! >>>>>> table drop_tables.c exists check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table test.snappy_finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 14:59:14 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.snappy_finish_mark not exists for 3-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 6.44 secs (513172702 bytes/sec) [Pipeline] { [Pipeline] cache The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.snappy_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6897.out cli changefeed pause -c snappy Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6917.out cli changefeed remove -c snappy start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: snappy CheckpointTs: 450114973014163462 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f22f40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t, pid:1160, start at 2024-05-30 14:59:20.916367384 +0800 CST m=+6.132837434 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:20.924 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:20.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:20.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6940.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f22f40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t, pid:1160, start at 2024-05-30 14:59:20.916367384 +0800 CST m=+6.132837434 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:20.924 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:20.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:20.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f269c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t, pid:1222, start at 2024-05-30 14:59:21.149705594 +0800 CST m=+6.189044302 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:21.156 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:21.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:21.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + set +x + tso='450114974324621315 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114974324621315 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6966.out cli changefeed create --start-ts=450114974324621315 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4' -c lz4 Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f31600018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:1137, start at 2024-05-30 14:59:21.858905183 +0800 CST m=+6.681328722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:21.869 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:21.867 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:21.867 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f31600018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:1137, start at 2024-05-30 14:59:21.858905183 +0800 CST m=+6.681328722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:21.869 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:21.867 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:21.867 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f3658000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:1202, start at 2024-05-30 14:59:22.146991057 +0800 CST m=+6.848901421 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:22.154 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:22.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:22.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Create changefeed successfully! ID: lz4 Info: {"upstream_id":7374683547363434342,"namespace":"default","id":"lz4","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=lz4","create_time":"2024-05-30T14:59:23.722547319+08:00","start_ts":450114974324621315,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114974324621315,"checkpoint_ts":450114974324621315,"checkpoint_time":"2024-05-30 14:59:21.773"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f30740004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:1194, start at 2024-05-30 14:59:21.761791863 +0800 CST m=+5.951724314 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:21.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:21.757 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:21.757 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f30740004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:1194, start at 2024-05-30 14:59:21.761791863 +0800 CST m=+5.951724314 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:21.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:21.757 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:21.757 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f31b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:1271, start at 2024-05-30 14:59:21.882606198 +0800 CST m=+5.979641847 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:21.891 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:21.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:21.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_basic_avro Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... + set +x [Thu May 30 14:59:25 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> table big_txn.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 14:59:23 CST 2024] <<<<<< run test case kafka_big_txn_v2 success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/30 14:59:23.681 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/30 14:59:23.717 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/30 14:59:23.814 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/30 14:59:23.825 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/30 14:59:24.815 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/30 14:59:24.825 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]: syntax error: operand expected (error token is "[2024/05/30 14:59:23.681 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/30 14:59:23.717 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/30 14:59:23.814 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/30 14:59:23.825 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/30 14:59:24.815 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/30 14:59:24.825 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]") table test.lz4_finish_mark not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 6.49 secs (509829963 bytes/sec) [Pipeline] { Starting Upstream TiDB... [Pipeline] cache Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2479.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f4c340012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:1263, start at 2024-05-30 14:59:23.564512368 +0800 CST m=+5.754565410 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:23.571 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:23.533 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:23.533 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f4c340012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:1263, start at 2024-05-30 14:59:23.564512368 +0800 CST m=+5.754565410 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:23.571 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:23.533 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:23.533 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f4db80011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:1328, start at 2024-05-30 14:59:23.653915574 +0800 CST m=+5.692153372 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:23.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:23.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:23.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Thu May 30 14:59:26 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.24922494.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f7cd40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k, pid:1203, start at 2024-05-30 14:59:26.675820188 +0800 CST m=+5.489566143 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:26.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:26.645 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:26.645 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.lz4_finish_mark not exists for 2-th check, retry later + set +x + tso='450114975463374849 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114975463374849 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 14:59:27 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.25102512.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Thu May 30 14:59:27 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/sourcemanager/puller/ProcessorAddTableError=1*return(true)' + etcd_info_msg='etcd info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.25462548.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2553.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f7cd40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k, pid:1203, start at 2024-05-30 14:59:26.675820188 +0800 CST m=+5.489566143 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:26.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:26.645 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:26.645 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212f7f140013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k, pid:1279, start at 2024-05-30 14:59:26.814154979 +0800 CST m=+5.486638369 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:26.820 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:26.789 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:26.789 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:59:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4275285a-daed-4619-abea-2722c8b4285c {"id":"4275285a-daed-4619-abea-2722c8b4285c","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b9d05be 4275285a-daed-4619-abea-2722c8b4285c /tidb/cdc/default/default/upstream/7374683698310877287 {"id":7374683698310877287,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4275285a-daed-4619-abea-2722c8b4285c {"id":"4275285a-daed-4619-abea-2722c8b4285c","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b9d05be 4275285a-daed-4619-abea-2722c8b4285c /tidb/cdc/default/default/upstream/7374683698310877287 {"id":7374683698310877287,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4275285a-daed-4619-abea-2722c8b4285c {"id":"4275285a-daed-4619-abea-2722c8b4285c","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b9d05be 4275285a-daed-4619-abea-2722c8b4285c /tidb/cdc/default/default/upstream/7374683698310877287 {"id":7374683698310877287,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Thu May 30 14:59:29 CST 2024] <<<<<< START kafka consumer in gc_safepoint case >>>>>> 0 table test.lz4_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7084.out cli changefeed pause -c lz4 + set +x + tso='450114976096976901 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114976096976901 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 14:59:30 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.25862588.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:59:30 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9596e3a6-dac8-498b-b6b7-468637fcf83f {"id":"9596e3a6-dac8-498b-b6b7-468637fcf83f","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b9e895a 9596e3a6-dac8-498b-b6b7-468637fcf83f /tidb/cdc/default/default/upstream/7374683692924779435 {"id":7374683692924779435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9596e3a6-dac8-498b-b6b7-468637fcf83f {"id":"9596e3a6-dac8-498b-b6b7-468637fcf83f","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b9e895a 9596e3a6-dac8-498b-b6b7-468637fcf83f /tidb/cdc/default/default/upstream/7374683692924779435 {"id":7374683692924779435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9596e3a6-dac8-498b-b6b7-468637fcf83f {"id":"9596e3a6-dac8-498b-b6b7-468637fcf83f","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84b9e895a 9596e3a6-dac8-498b-b6b7-468637fcf83f /tidb/cdc/default/default/upstream/7374683692924779435 {"id":7374683692924779435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:59:30 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88ba01b1-d624-4b7b-aece-9a87a61fb8be {"id":"88ba01b1-d624-4b7b-aece-9a87a61fb8be","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ba004c7 88ba01b1-d624-4b7b-aece-9a87a61fb8be /tidb/cdc/default/default/upstream/7374683698613802498 {"id":7374683698613802498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88ba01b1-d624-4b7b-aece-9a87a61fb8be {"id":"88ba01b1-d624-4b7b-aece-9a87a61fb8be","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ba004c7 88ba01b1-d624-4b7b-aece-9a87a61fb8be /tidb/cdc/default/default/upstream/7374683698613802498 {"id":7374683698613802498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88ba01b1-d624-4b7b-aece-9a87a61fb8be {"id":"88ba01b1-d624-4b7b-aece-9a87a61fb8be","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ba004c7 88ba01b1-d624-4b7b-aece-9a87a61fb8be /tidb/cdc/default/default/upstream/7374683698613802498 {"id":7374683698613802498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2556.out cli changefeed create --start-ts=450114975463374849 '--sink-uri=kafka://127.0.0.1:9092/ticdc-foreign-key-test-24715?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7116.out cli changefeed remove -c lz4 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Create changefeed successfully! ID: 9a6567a8-59df-4d08-a1ae-bbf6094f9b33 Info: {"upstream_id":7374683698613802498,"namespace":"default","id":"9a6567a8-59df-4d08-a1ae-bbf6094f9b33","sink_uri":"kafka://127.0.0.1:9092/ticdc-foreign-key-test-24715?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T14:59:31.522578942+08:00","start_ts":450114975463374849,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114975463374849,"checkpoint_ts":450114975463374849,"checkpoint_time":"2024-05-30 14:59:26.117"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Thu May 30 14:59:31 CST 2024] <<<<<< START kafka consumer in processor_err_chan case >>>>>> check_changefeed_state http://127.0.0.1:2379 386fc98d-93e9-4d3f-93e2-99dca30d1be9 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=386fc98d-93e9-4d3f-93e2-99dca30d1be9 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 386fc98d-93e9-4d3f-93e2-99dca30d1be9 -s [Thu May 30 14:59:31 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.25212523.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7374683692924779435, "namespace": "default", "id": "386fc98d-93e9-4d3f-93e2-99dca30d1be9", "state": "normal", "checkpoint_tso": 450114976754958340, "checkpoint_time": "2024-05-30 14:59:31.044", "error": null }' + echo '{ "upstream_id": 7374683692924779435, "namespace": "default", "id": "386fc98d-93e9-4d3f-93e2-99dca30d1be9", "state": "normal", "checkpoint_tso": 450114976754958340, "checkpoint_time": "2024-05-30 14:59:31.044", "error": null }' { "upstream_id": 7374683692924779435, "namespace": "default", "id": "386fc98d-93e9-4d3f-93e2-99dca30d1be9", "state": "normal", "checkpoint_tso": 450114976754958340, "checkpoint_time": "2024-05-30 14:59:31.044", "error": null } ++ echo '{' '"upstream_id":' 7374683692924779435, '"namespace":' '"default",' '"id":' '"386fc98d-93e9-4d3f-93e2-99dca30d1be9",' '"state":' '"normal",' '"checkpoint_tso":' 450114976754958340, '"checkpoint_time":' '"2024-05-30' '14:59:31.044",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374683692924779435, '"namespace":' '"default",' '"id":' '"386fc98d-93e9-4d3f-93e2-99dca30d1be9",' '"state":' '"normal",' '"checkpoint_tso":' 450114976754958340, '"checkpoint_time":' '"2024-05-30' '14:59:31.044",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Changefeed remove successfully. ID: lz4 CheckpointTs: 450114975242125343 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later + set +x [Thu May 30 14:59:33 CST 2024] <<<<<< START kafka consumer in foreign_key case >>>>>> check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:59:33 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/361ae268-3eb1-478d-b268-8ddefc8d7d0d {"id":"361ae268-3eb1-478d-b268-8ddefc8d7d0d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84baa9ac2 361ae268-3eb1-478d-b268-8ddefc8d7d0d /tidb/cdc/default/default/upstream/7374683702768697772 {"id":7374683702768697772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/361ae268-3eb1-478d-b268-8ddefc8d7d0d {"id":"361ae268-3eb1-478d-b268-8ddefc8d7d0d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84baa9ac2 361ae268-3eb1-478d-b268-8ddefc8d7d0d /tidb/cdc/default/default/upstream/7374683702768697772 {"id":7374683702768697772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/361ae268-3eb1-478d-b268-8ddefc8d7d0d {"id":"361ae268-3eb1-478d-b268-8ddefc8d7d0d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84baa9ac2 361ae268-3eb1-478d-b268-8ddefc8d7d0d /tidb/cdc/default/default/upstream/7374683702768697772 {"id":7374683702768697772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2632.out cli changefeed create --start-ts=450114976096976901 '--sink-uri=kafka://127.0.0.1:9092/ticdc-resolve-lock-test-8825?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: e5fead10-51bd-4a79-b8fb-2e72fe5042ad Info: {"upstream_id":7374683702768697772,"namespace":"default","id":"e5fead10-51bd-4a79-b8fb-2e72fe5042ad","sink_uri":"kafka://127.0.0.1:9092/ticdc-resolve-lock-test-8825?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T14:59:33.628813591+08:00","start_ts":450114976096976901,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114976096976901,"checkpoint_ts":450114976096976901,"checkpoint_time":"2024-05-30 14:59:28.534"} PASS + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7138.out cli tso query --pd=http://127.0.0.1:2379 coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212fd108000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:1250, start at 2024-05-30 14:59:32.052911641 +0800 CST m=+6.248135675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:32.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:32.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:32.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212fd108000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:1250, start at 2024-05-30 14:59:32.052911641 +0800 CST m=+6.248135675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:32.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:32.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:32.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f212fd1b80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:1317, start at 2024-05-30 14:59:32.127350602 +0800 CST m=+6.209474598 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:32.134 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:32.128 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:32.128 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:59:34 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af640051-3263-4e42-bdff-940ea08122cd {"id":"af640051-3263-4e42-bdff-940ea08122cd","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bb406be af640051-3263-4e42-bdff-940ea08122cd /tidb/cdc/default/default/upstream/7374683725725976498 {"id":7374683725725976498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af640051-3263-4e42-bdff-940ea08122cd {"id":"af640051-3263-4e42-bdff-940ea08122cd","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bb406be af640051-3263-4e42-bdff-940ea08122cd /tidb/cdc/default/default/upstream/7374683725725976498 {"id":7374683725725976498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af640051-3263-4e42-bdff-940ea08122cd {"id":"af640051-3263-4e42-bdff-940ea08122cd","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bb406be af640051-3263-4e42-bdff-940ea08122cd /tidb/cdc/default/default/upstream/7374683725725976498 {"id":7374683725725976498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Thu May 30 14:59:35 CST 2024] <<<<<< START kafka consumer in resolve_lock case >>>>>> go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240510070134-e7f530c64790 go: downloading github.com/tikv/pd/client v0.0.0-20240210135946-3488a653ddd9 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240527072219-a97e6464c01d go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240527072219-a97e6464c01d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/kvproto v0.0.0-20240522024016-df42997c2c57 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 check diff failed 2-th time, retry later go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully check_safepoint_forward http://127.0.0.1:2379 7374683698310877287 450114977518583816 450114976338935810 + set +x + tso='450114977535885315 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114977535885315 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7175.out cli changefeed create --start-ts=450114977535885315 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd' -c zstd go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/prometheus/client_golang v1.19.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading google.golang.org/protobuf v1.34.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.15.0 Create changefeed successfully! ID: zstd Info: {"upstream_id":7374683547363434342,"namespace":"default","id":"zstd","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=zstd","create_time":"2024-05-30T14:59:36.092553326+08:00","start_ts":450114977535885315,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114977535885315,"checkpoint_ts":450114977535885315,"checkpoint_time":"2024-05-30 14:59:34.023"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... go: downloading golang.org/x/sys v0.20.0 table foreign_key.finish_mark not exists for 1-th check, retry later go: downloading golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 go: downloading github.com/cloudfoundry/gosigar v1.3.6 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 7.31 secs (452186729 bytes/sec) [Pipeline] { [Pipeline] cache go: downloading golang.org/x/net v0.25.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2545.out cli tso query --pd=http://127.0.0.1:2379 run task successfully go: downloading golang.org/x/text v0.15.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... check_changefeed_state http://127.0.0.1:2379 fcb22880-16ce-46b5-a90b-ef29dbc27921 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=fcb22880-16ce-46b5-a90b-ef29dbc27921 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c fcb22880-16ce-46b5-a90b-ef29dbc27921 -s + set +x table foreign_key.finish_mark not exists for 2-th check, retry later Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Thu May 30 14:59:37 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> + set +x + tso='450114978220343297 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114978220343297 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 14:59:38 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.25722574.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/30 14:59:35.991 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/30 14:59:36.087 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/30 14:59:36.222 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/30 14:59:36.232 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/30 14:59:37.222 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/30 14:59:37.233 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]: syntax error: operand expected (error token is "[2024/05/30 14:59:35.991 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/30 14:59:36.087 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/30 14:59:36.222 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/30 14:59:36.232 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/30 14:59:37.222 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/30 14:59:37.233 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]") table test.zstd_finish_mark not exists for 1-th check, retry later + info='{ "upstream_id": 7374683698310877287, "namespace": "default", "id": "fcb22880-16ce-46b5-a90b-ef29dbc27921", "state": "stopped", "checkpoint_tso": 450114978042871815, "checkpoint_time": "2024-05-30 14:59:35.957", "error": null }' + echo '{ "upstream_id": 7374683698310877287, "namespace": "default", "id": "fcb22880-16ce-46b5-a90b-ef29dbc27921", "state": "stopped", "checkpoint_tso": 450114978042871815, "checkpoint_time": "2024-05-30 14:59:35.957", "error": null }' { "upstream_id": 7374683698310877287, "namespace": "default", "id": "fcb22880-16ce-46b5-a90b-ef29dbc27921", "state": "stopped", "checkpoint_tso": 450114978042871815, "checkpoint_time": "2024-05-30 14:59:35.957", "error": null } go: downloading github.com/pingcap/tipb v0.0.0-20240507090649-2bf6bb0cb996 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/stretchr/testify v1.9.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading golang.org/x/tools v0.21.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/sasha-s/go-deadlock v0.3.1 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.20.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.6 go: downloading github.com/robfig/cron v1.2.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 ++ echo '{' '"upstream_id":' 7374683698310877287, '"namespace":' '"default",' '"id":' '"fcb22880-16ce-46b5-a90b-ef29dbc27921",' '"state":' '"stopped",' '"checkpoint_tso":' 450114978042871815, '"checkpoint_time":' '"2024-05-30' '14:59:35.957",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7374683698310877287, '"namespace":' '"default",' '"id":' '"fcb22880-16ce-46b5-a90b-ef29dbc27921",' '"state":' '"stopped",' '"checkpoint_tso":' 450114978042871815, '"checkpoint_time":' '"2024-05-30' '14:59:35.957",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7374683698310877287 start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages_v2 Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.23.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/apache/thrift v0.16.0 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/rivo/uniseg v0.4.7 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21301da4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:4084, start at 2024-05-30 14:59:36.951680334 +0800 CST m=+6.253592376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:36.960 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:36.937 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:36.937 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21301da4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:4084, start at 2024-05-30 14:59:36.951680334 +0800 CST m=+6.253592376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:36.960 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:36.937 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:36.937 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21302194000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:4161, start at 2024-05-30 14:59:37.204594922 +0800 CST m=+6.402709245 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:37.213 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:37.189 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:37.189 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/tklauser/numcpus v0.6.1 check diff successfully wait process cdc.test exit for 1-th time... go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac table foreign_key.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21304e0c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:6083, start at 2024-05-30 14:59:40.056916038 +0800 CST m=+6.380456543 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:40.068 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:40.035 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:40.035 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/DataDog/zstd v1.5.5 wait process cdc.test exit for 3-th time... go: downloading k8s.io/apimachinery v0.28.6 table test.zstd_finish_mark not exists for 2-th check, retry later [Thu May 30 14:59:41 CST 2024] <<<<<< START kafka consumer in kv_client_stream_reconnect case >>>>>> cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 14:59:41 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> [Thu May 30 14:59:41 CST 2024] <<<<<< START cdc server in kafka_simple_basic_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic_avro.53805382.out server --log-file /tmp/tidb_cdc_test/kafka_simple_basic_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_basic_avro/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b table foreign_key.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21304e0c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:6083, start at 2024-05-30 14:59:40.056916038 +0800 CST m=+6.380456543 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:40.068 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:40.035 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:40.035 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213050800014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:6157, start at 2024-05-30 14:59:40.23636383 +0800 CST m=+6.438683454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:40.244 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:40.242 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:40.242 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:59:42 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbcd3099-1abd-4818-9750-0d1a4d22e603 {"id":"dbcd3099-1abd-4818-9750-0d1a4d22e603","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bc3b6c8 dbcd3099-1abd-4818-9750-0d1a4d22e603 /tidb/cdc/default/default/upstream/7374683739104673313 {"id":7374683739104673313,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbcd3099-1abd-4818-9750-0d1a4d22e603 {"id":"dbcd3099-1abd-4818-9750-0d1a4d22e603","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bc3b6c8 dbcd3099-1abd-4818-9750-0d1a4d22e603 /tidb/cdc/default/default/upstream/7374683739104673313 {"id":7374683739104673313,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check_changefeed_state http://127.0.0.1:2379 fcb22880-16ce-46b5-a90b-ef29dbc27921 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=fcb22880-16ce-46b5-a90b-ef29dbc27921 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c fcb22880-16ce-46b5-a90b-ef29dbc27921 -s + info='{ "upstream_id": 7374683698310877287, "namespace": "default", "id": "fcb22880-16ce-46b5-a90b-ef29dbc27921", "state": "normal", "checkpoint_tso": 450114978042871815, "checkpoint_time": "2024-05-30 14:59:35.957", "error": null }' + echo '{ "upstream_id": 7374683698310877287, "namespace": "default", "id": "fcb22880-16ce-46b5-a90b-ef29dbc27921", "state": "normal", "checkpoint_tso": 450114978042871815, "checkpoint_time": "2024-05-30 14:59:35.957", "error": null }' { "upstream_id": 7374683698310877287, "namespace": "default", "id": "fcb22880-16ce-46b5-a90b-ef29dbc27921", "state": "normal", "checkpoint_tso": 450114978042871815, "checkpoint_time": "2024-05-30 14:59:35.957", "error": null } ++ echo '{' '"upstream_id":' 7374683698310877287, '"namespace":' '"default",' '"id":' '"fcb22880-16ce-46b5-a90b-ef29dbc27921",' '"state":' '"normal",' '"checkpoint_tso":' 450114978042871815, '"checkpoint_time":' '"2024-05-30' '14:59:35.957",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374683698310877287, '"namespace":' '"default",' '"id":' '"fcb22880-16ce-46b5-a90b-ef29dbc27921",' '"state":' '"normal",' '"checkpoint_tso":' 450114978042871815, '"checkpoint_time":' '"2024-05-30' '14:59:35.957",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7374683698310877287 450114978042871814 450114978042871815 table test.zstd_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7272.out cli changefeed pause -c zstd PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... run task successfully table foreign_key.finish_mark not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 fcb22880-16ce-46b5-a90b-ef29dbc27921 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=fcb22880-16ce-46b5-a90b-ef29dbc27921 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c fcb22880-16ce-46b5-a90b-ef29dbc27921 -s + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbcd3099-1abd-4818-9750-0d1a4d22e603 {"id":"dbcd3099-1abd-4818-9750-0d1a4d22e603","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bc3b6c8 dbcd3099-1abd-4818-9750-0d1a4d22e603 /tidb/cdc/default/default/upstream/7374683739104673313 {"id":7374683739104673313,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2626.out cli changefeed create --start-ts=450114978220343297 '--sink-uri=kafka://127.0.0.1:9092/ticdc-default-value-test-30960?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + info='{ "upstream_id": 7374683698310877287, "namespace": "default", "id": "fcb22880-16ce-46b5-a90b-ef29dbc27921", "state": "stopped", "checkpoint_tso": 450114978042871815, "checkpoint_time": "2024-05-30 14:59:35.957", "error": null }' + echo '{ "upstream_id": 7374683698310877287, "namespace": "default", "id": "fcb22880-16ce-46b5-a90b-ef29dbc27921", "state": "stopped", "checkpoint_tso": 450114978042871815, "checkpoint_time": "2024-05-30 14:59:35.957", "error": null }' { "upstream_id": 7374683698310877287, "namespace": "default", "id": "fcb22880-16ce-46b5-a90b-ef29dbc27921", "state": "stopped", "checkpoint_tso": 450114978042871815, "checkpoint_time": "2024-05-30 14:59:35.957", "error": null } ++ echo '{' '"upstream_id":' 7374683698310877287, '"namespace":' '"default",' '"id":' '"fcb22880-16ce-46b5-a90b-ef29dbc27921",' '"state":' '"stopped",' '"checkpoint_tso":' 450114978042871815, '"checkpoint_time":' '"2024-05-30' '14:59:35.957",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7374683698310877287, '"namespace":' '"default",' '"id":' '"fcb22880-16ce-46b5-a90b-ef29dbc27921",' '"state":' '"stopped",' '"checkpoint_tso":' 450114978042871815, '"checkpoint_time":' '"2024-05-30' '14:59:35.957",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully [Thu May 30 14:59:44 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.74287430.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' check_changefeed_state http://127.0.0.1:2379 6895d1a6-1b44-4534-b537-85e6a29c6db7 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=6895d1a6-1b44-4534-b537-85e6a29c6db7 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 6895d1a6-1b44-4534-b537-85e6a29c6db7 -s + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:59:44 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21036215-608d-4a86-ab7c-484c7dc6ce57 {"id":"21036215-608d-4a86-ab7c-484c7dc6ce57","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bd726ca 21036215-608d-4a86-ab7c-484c7dc6ce57 /tidb/cdc/default/default/upstream/7374683763390049273 {"id":7374683763390049273,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21036215-608d-4a86-ab7c-484c7dc6ce57 {"id":"21036215-608d-4a86-ab7c-484c7dc6ce57","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bd726ca 21036215-608d-4a86-ab7c-484c7dc6ce57 /tidb/cdc/default/default/upstream/7374683763390049273 {"id":7374683763390049273,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21036215-608d-4a86-ab7c-484c7dc6ce57 {"id":"21036215-608d-4a86-ab7c-484c7dc6ce57","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bd726ca 21036215-608d-4a86-ab7c-484c7dc6ce57 /tidb/cdc/default/default/upstream/7374683763390049273 {"id":7374683763390049273,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic_avro.cli.5421.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-simple-basic-avro-7477?protocol=simple&encoding-format=avro' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/conf/changefeed.toml Create changefeed successfully! ID: 7a8e5060-4a2e-4637-bde8-c1d073747732 Info: {"upstream_id":7374683739104673313,"namespace":"default","id":"7a8e5060-4a2e-4637-bde8-c1d073747732","sink_uri":"kafka://127.0.0.1:9092/ticdc-default-value-test-30960?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T14:59:44.611219445+08:00","start_ts":450114978220343297,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114978220343297,"checkpoint_ts":450114978220343297,"checkpoint_time":"2024-05-30 14:59:36.634"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7295.out cli changefeed remove -c zstd + info='{ "upstream_id": 7374683698310877287, "namespace": "default", "id": "6895d1a6-1b44-4534-b537-85e6a29c6db7", "state": "normal", "checkpoint_tso": 450114980284203013, "checkpoint_time": "2024-05-30 14:59:44.507", "error": null }' + echo '{ "upstream_id": 7374683698310877287, "namespace": "default", "id": "6895d1a6-1b44-4534-b537-85e6a29c6db7", "state": "normal", "checkpoint_tso": 450114980284203013, "checkpoint_time": "2024-05-30 14:59:44.507", "error": null }' { "upstream_id": 7374683698310877287, "namespace": "default", "id": "6895d1a6-1b44-4534-b537-85e6a29c6db7", "state": "normal", "checkpoint_tso": 450114980284203013, "checkpoint_time": "2024-05-30 14:59:44.507", "error": null } ++ echo '{' '"upstream_id":' 7374683698310877287, '"namespace":' '"default",' '"id":' '"6895d1a6-1b44-4534-b537-85e6a29c6db7",' '"state":' '"normal",' '"checkpoint_tso":' 450114980284203013, '"checkpoint_time":' '"2024-05-30' '14:59:44.507",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374683698310877287, '"namespace":' '"default",' '"id":' '"6895d1a6-1b44-4534-b537-85e6a29c6db7",' '"state":' '"normal",' '"checkpoint_tso":' 450114980284203013, '"checkpoint_time":' '"2024-05-30' '14:59:44.507",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7374683698310877287 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 465f368d-4ac5-494f-938f-5c077a2c27da Info: {"upstream_id":7374683763390049273,"namespace":"default","id":"465f368d-4ac5-494f-938f-5c077a2c27da","sink_uri":"kafka://127.0.0.1:9092/ticdc-simple-basic-avro-7477?protocol=simple\u0026encoding-format=avro","create_time":"2024-05-30T14:59:45.156527308+08:00","start_ts":450114980410032134,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114980410032134,"checkpoint_ts":450114980410032134,"checkpoint_time":"2024-05-30 14:59:44.987"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Changefeed remove successfully. ID: zstd CheckpointTs: 450114978728640537 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table foreign_key.finish_mark not exists for 6-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21309d08000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:3684, start at 2024-05-30 14:59:45.104732802 +0800 CST m=+7.483962231 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:45.115 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:45.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:45.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Thu May 30 14:59:46 CST 2024] <<<<<< START kafka consumer in default_value case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240527072219-a97e6464c01d go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading golang.org/x/time v0.5.0 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240527072219-a97e6464c01d go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading golang.org/x/sys v0.20.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e go: downloading google.golang.org/protobuf v1.34.1 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda + set +x wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21309d08000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:3684, start at 2024-05-30 14:59:45.104732802 +0800 CST m=+7.483962231 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:45.115 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:45.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:45.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213088c80011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:3749, start at 2024-05-30 14:59:43.826625986 +0800 CST m=+6.094730421 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:43.834 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:43.794 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:43.794 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table foreign_key.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:59:47 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c42cc99-bd98-4cce-ae0f-5e2bee7a866d {"id":"8c42cc99-bd98-4cce-ae0f-5e2bee7a866d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84be885c7 8c42cc99-bd98-4cce-ae0f-5e2bee7a866d /tidb/cdc/default/default/upstream/7374683777402213226 {"id":7374683777402213226,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c42cc99-bd98-4cce-ae0f-5e2bee7a866d {"id":"8c42cc99-bd98-4cce-ae0f-5e2bee7a866d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84be885c7 8c42cc99-bd98-4cce-ae0f-5e2bee7a866d /tidb/cdc/default/default/upstream/7374683777402213226 {"id":7374683777402213226,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8c42cc99-bd98-4cce-ae0f-5e2bee7a866d {"id":"8c42cc99-bd98-4cce-ae0f-5e2bee7a866d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84be885c7 8c42cc99-bd98-4cce-ae0f-5e2bee7a866d /tidb/cdc/default/default/upstream/7374683777402213226 {"id":7374683777402213226,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Thu May 30 14:59:47 CST 2024] <<<<<< START kafka consumer in processor_stop_delay case >>>>>> Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } run task successfully table processor_stop_delay.t not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 8.00 secs (413134264 bytes/sec) [Pipeline] { [Pipeline] cache go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 Changefeed remove successfully. ID: fcb22880-16ce-46b5-a90b-ef29dbc27921 CheckpointTs: 450114978042871815 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-4825?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 check_safepoint_forward http://127.0.0.1:2379 7374683698310877287 450114978042871814 450114980284203013 450114978042871815 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 14:59:49 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> run task successfully table foreign_key.finish_mark not exists for 8-th check, retry later Changefeed remove successfully. ID: 6895d1a6-1b44-4534-b537-85e6a29c6db7 CheckpointTs: 450114981463851009 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-4825?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 check_safepoint_cleared http://127.0.0.1:2379 7374683698310877287 run task successfully table processor_stop_delay.t not exists for 2-th check, retry later go: downloading github.com/pingcap/kvproto v0.0.0-20240522024016-df42997c2c57 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/text v0.15.0 go: downloading github.com/pingcap/tipb v0.0.0-20240507090649-2bf6bb0cb996 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/google/btree v1.1.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 go: downloading golang.org/x/net v0.25.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240510070134-e7f530c64790 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading github.com/tikv/pd/client v0.0.0-20240210135946-3488a653ddd9 go: downloading github.com/prometheus/client_golang v1.19.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading golang.org/x/tools v0.21.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.6 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/spkg/bom v1.0.0 wait process cdc.test exit for 1-th time... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.4981.out cli tso query --pd=http://127.0.0.1:2379 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/prometheus/common v0.53.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/beorn7/perks v1.0.1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2130e0e8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:4666, start at 2024-05-30 14:59:49.454704928 +0800 CST m=+7.070518204 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:49.480 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:49.483 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:49.483 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2130e0e8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:4666, start at 2024-05-30 14:59:49.454704928 +0800 CST m=+7.070518204 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:49.480 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:49.483 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:49.483 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2130e26c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:4736, start at 2024-05-30 14:59:49.539310313 +0800 CST m=+7.040448766 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:01:49.546 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-14:59:49.531 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:49:49.531 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading github.com/prometheus/procfs v0.15.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 wait process cdc.test exit for 2-th time... go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/sasha-s/go-deadlock v0.3.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/json-iterator/go v1.1.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 wait process cdc.test exit for 3-th time... go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67 + set +x + tso='450114981826658305 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114981826658305 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 14:59:51 CST 2024] <<<<<< START kafka consumer in kafka_simple_basic_avro case >>>>>> go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 ***************** properties ***************** "updateproportion"="0" "dotransactions"="false" "recordcount"="20" "mysql.port"="4000" "mysql.host"="127.0.0.1" "mysql.user"="root" "threadcount"="4" "operationcount"="0" "insertproportion"="0" "mysql.db"="changefeed_error" "readallfields"="true" "scanproportion"="0" "requestdistribution"="uniform" "workload"="core" "readproportion"="0" ********************************************** Run finished, takes 25.605029ms INSERT - Takes(s): 0.0, Count: 18, OPS: 3092.9, Avg(us): 3349, Min(us): 1046, Max(us): 19878, 95th(us): 20000, 99th(us): 20000 [Thu May 30 14:59:52 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.50425044.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 9-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 14:59:52 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 table processor_stop_delay.t exists check diff failed 1-th time, retry later go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/DataDog/zstd v1.5.5 [Thu May 30 14:59:52 CST 2024] <<<<<< START cdc server in kafka_big_messages_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages_v2.59735975.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table foreign_key.finish_mark not exists for 10-th check, retry later check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:59:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 {"id":"f28c4436-d960-458f-a7cd-9beef8506b20","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf21ee2 f28c4436-d960-458f-a7cd-9beef8506b20 /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 {"id":"f28c4436-d960-458f-a7cd-9beef8506b20","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf21ee2 f28c4436-d960-458f-a7cd-9beef8506b20 /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 {"id":"f28c4436-d960-458f-a7cd-9beef8506b20","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf21ee2 f28c4436-d960-458f-a7cd-9beef8506b20 /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5088.out cli changefeed create --start-ts=450114981826658305 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7374683796122836486,"namespace":"default","id":"changefeed-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T14:59:55.834510807+08:00","start_ts":450114981826658305,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114981826658305,"checkpoint_ts":450114981826658305,"checkpoint_time":"2024-05-30 14:59:50.391"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table foreign_key.finish_mark exists check diff successfully check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 06:59:56 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0698020-0f22-4a1f-b7fd-3fbc42645525 {"id":"d0698020-0f22-4a1f-b7fd-3fbc42645525","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c105dc6 d0698020-0f22-4a1f-b7fd-3fbc42645525 /tidb/cdc/default/default/upstream/7374683815418502652 {"id":7374683815418502652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0698020-0f22-4a1f-b7fd-3fbc42645525 {"id":"d0698020-0f22-4a1f-b7fd-3fbc42645525","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c105dc6 d0698020-0f22-4a1f-b7fd-3fbc42645525 /tidb/cdc/default/default/upstream/7374683815418502652 {"id":7374683815418502652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0698020-0f22-4a1f-b7fd-3fbc42645525 {"id":"d0698020-0f22-4a1f-b7fd-3fbc42645525","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c105dc6 d0698020-0f22-4a1f-b7fd-3fbc42645525 /tidb/cdc/default/default/upstream/7374683815418502652 {"id":7374683815418502652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 26a34882-fda3-40b4-8d90-396566c80205 Info: {"upstream_id":7374683815418502652,"namespace":"default","id":"26a34882-fda3-40b4-8d90-396566c80205","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-05-30T14:59:56.227382556+08:00","start_ts":450114982480183297,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114982480183297,"checkpoint_ts":450114982480183297,"checkpoint_time":"2024-05-30 14:59:52.884"} [Thu May 30 14:59:56 CST 2024] <<<<<< START kafka consumer in kafka_big_messages_v2 case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/atomic v1.11.0 wait process cdc.test exit for 1-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 5.97 secs (554142661 bytes/sec) [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... go: downloading github.com/jmespath/go-jmespath v0.4.0 wait process cdc.test exit for 2-th time... go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + set +x [Thu May 30 14:59:57 CST 2024] <<<<<< START kafka consumer in changefeed_error case >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s wait process cdc.test exit for 3-th time... + info='{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450114981826658305, "checkpoint_time": "2024-05-30 14:59:50.391", "error": { "time": "2024-05-30T14:59:55.931656991+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450114981826658005 is earlier than or equal to GC safepoint at 450114981826658305" } }' + echo '{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450114981826658305, "checkpoint_time": "2024-05-30 14:59:50.391", "error": { "time": "2024-05-30T14:59:55.931656991+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450114981826658005 is earlier than or equal to GC safepoint at 450114981826658305" } }' { "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450114981826658305, "checkpoint_time": "2024-05-30 14:59:50.391", "error": { "time": "2024-05-30T14:59:55.931656991+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450114981826658005 is earlier than or equal to GC safepoint at 450114981826658305" } } ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 450114981826658305, '"checkpoint_time":' '"2024-05-30' '14:59:50.391",' '"error":' '{' '"time":' '"2024-05-30T14:59:55.931656991+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 450114981826658005 is earlier than or equal to GC safepoint at '450114981826658305"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 450114981826658305, '"checkpoint_time":' '"2024-05-30' '14:59:50.391",' '"error":' '{' '"time":' '"2024-05-30T14:59:55.931656991+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 450114981826658005 is earlier than or equal to GC safepoint at '450114981826658305"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450114981826658005 is earlier than or equal to GC safepoint at 450114981826658305' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450114981826658005 is earlier than or equal to GC safepoint at 450114981826658305 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5188.out cli changefeed resume -c changefeed-error Verifying downstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 14:59:57 CST 2024] <<<<<< run test case foreign_key success! >>>>>> check diff failed 2-th time, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release PASS check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore coverage: 2.0% of statements in github.com/pingcap/tiflow/... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:00:00 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> + set +x table changefeed_error.usertable not exists for 1-th check, retry later check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_error.usertable exists =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 1-th time, retry later table kafka_big_messages.test exists check diff failed 1-th time, retry later check diff failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully ***************** properties ***************** "recordcount"="20" "mysql.db"="changefeed_error" "scanproportion"="0" "insertproportion"="0" "threadcount"="4" "mysql.user"="root" "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.port"="4000" "dotransactions"="false" "readproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "workload"="core" "updateproportion"="0" ********************************************** Run finished, takes 5.194641ms INSERT - Takes(s): 0.0, Count: 20, OPS: 6327.7, Avg(us): 948, Min(us): 619, Max(us): 2101, 95th(us): 3000, 99th(us): 3000 check diff successfully {"id":"f28c4436-d960-458f-a7cd-9beef8506b20","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 + [[ /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/kafka_sink_error_resume Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... check diff failed 2-th time, retry later check diff failed 6-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2131f3c00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t, pid:3805, start at 2024-05-30 15:00:07.071997838 +0800 CST m=+6.089476672 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:07.079 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:07.073 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:07.073 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully check diff successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 + [[ /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2131f3c00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t, pid:3805, start at 2024-05-30 15:00:07.071997838 +0800 CST m=+6.089476672 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:07.079 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:07.073 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:07.073 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2131f5540015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t, pid:3859, start at 2024-05-30 15:00:07.169997222 +0800 CST m=+6.020045833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:07.177 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:07.175 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:07.175 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:00:10 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> wait process cdc.test exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:00:11 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> table test.finish_mark not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Thu May 30 15:00:11 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.50595061.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 4-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 + [[ /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/f28c4436-d960-458f-a7cd-9beef8506b20 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:00:14 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e0bcc81-2f1a-4a82-bc52-b26f8787c7ac {"id":"4e0bcc81-2f1a-4a82-bc52-b26f8787c7ac","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c531ec4 4e0bcc81-2f1a-4a82-bc52-b26f8787c7ac /tidb/cdc/default/default/upstream/7374683888318620092 {"id":7374683888318620092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e0bcc81-2f1a-4a82-bc52-b26f8787c7ac {"id":"4e0bcc81-2f1a-4a82-bc52-b26f8787c7ac","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c531ec4 4e0bcc81-2f1a-4a82-bc52-b26f8787c7ac /tidb/cdc/default/default/upstream/7374683888318620092 {"id":7374683888318620092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e0bcc81-2f1a-4a82-bc52-b26f8787c7ac {"id":"4e0bcc81-2f1a-4a82-bc52-b26f8787c7ac","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c531ec4 4e0bcc81-2f1a-4a82-bc52-b26f8787c7ac /tidb/cdc/default/default/upstream/7374683888318620092 {"id":7374683888318620092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 5-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Thu May 30 15:00:15 CST 2024] <<<<<< START kafka consumer in changefeed_reconstruct case >>>>>> ***************** properties ***************** "workload"="core" "readallfields"="true" "operationcount"="0" "mysql.db"="changefeed_reconstruct" "mysql.user"="root" "scanproportion"="0" "dotransactions"="false" "readproportion"="0" "insertproportion"="0" "threadcount"="4" "mysql.port"="4000" "requestdistribution"="uniform" "updateproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="50" ********************************************** Run finished, takes 21.74976ms INSERT - Takes(s): 0.0, Count: 48, OPS: 2873.1, Avg(us): 1651, Min(us): 1011, Max(us): 4978, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 18.18 secs (181863422 bytes/sec) [Pipeline] { [Pipeline] cache table test.finish_mark not exists for 6-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2132a64c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:8306, start at 2024-05-30 15:00:18.473737519 +0800 CST m=+7.157522788 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:18.483 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:18.451 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:18.451 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2132a64c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:8306, start at 2024-05-30 15:00:18.473737519 +0800 CST m=+7.157522788 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:18.483 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:18.451 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:18.451 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2132a8880013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:8374, start at 2024-05-30 15:00:18.628186303 +0800 CST m=+7.153229001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:18.635 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:18.594 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:18.594 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Thu May 30 15:00:18 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.54205422.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table changefeed_reconstruct.usertable exists check diff successfully wait process 5064 exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:00:20 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> wait process 5064 exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2132b7480012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:4177, start at 2024-05-30 15:00:19.562696695 +0800 CST m=+6.750240071 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:19.572 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:19.538 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:19.538 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2132b7480012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:4177, start at 2024-05-30 15:00:19.562696695 +0800 CST m=+6.750240071 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:19.572 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:19.538 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:19.538 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2132b8180014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:4248, start at 2024-05-30 15:00:19.638737144 +0800 CST m=+6.663133268 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:19.646 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:19.640 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:19.640 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process 5064 exit for 3-th time... table test.finish_mark not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:00:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5dde4d3-6b7e-4b9d-bb0b-efdb6cb04bdd {"id":"f5dde4d3-6b7e-4b9d-bb0b-efdb6cb04bdd","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf21fba f5dde4d3-6b7e-4b9d-bb0b-efdb6cb04bdd /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7374683796122836486,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T14:59:55.834510807+08:00","start-ts":450114981826658305,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-30T15:00:19.450749187+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450114989441679362} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450114985850568707,"min-table-barrier-ts":450114985850568708,"admin-job-type":1} /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5dde4d3-6b7e-4b9d-bb0b-efdb6cb04bdd {"id":"f5dde4d3-6b7e-4b9d-bb0b-efdb6cb04bdd","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf21fba f5dde4d3-6b7e-4b9d-bb0b-efdb6cb04bdd /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7374683796122836486,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T14:59:55.834510807+08:00","start-ts":450114981826658305,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-30T15:00:19.450749187+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450114989441679362} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450114985850568707,"min-table-barrier-ts":450114985850568708,"admin-job-type":1} /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5dde4d3-6b7e-4b9d-bb0b-efdb6cb04bdd {"id":"f5dde4d3-6b7e-4b9d-bb0b-efdb6cb04bdd","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf21fba f5dde4d3-6b7e-4b9d-bb0b-efdb6cb04bdd /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7374683796122836486,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T14:59:55.834510807+08:00","start-ts":450114981826658305,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-30T15:00:19.450749187+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450114989441679362} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450114985850568707,"min-table-barrier-ts":450114985850568708,"admin-job-type":1} /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5064) - No such process wait process 5064 exit for 4-th time... process 5064 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Thu May 30 15:00:22 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.52555257.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + info='{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450114985850568707, "checkpoint_time": "2024-05-30 15:00:05.741", "error": { "time": "2024-05-30T15:00:19.450749187+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450114985850568707, "checkpoint_time": "2024-05-30 15:00:05.741", "error": { "time": "2024-05-30T15:00:19.450749187+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450114985850568707, "checkpoint_time": "2024-05-30 15:00:05.741", "error": { "time": "2024-05-30T15:00:19.450749187+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 450114985850568707, '"checkpoint_time":' '"2024-05-30' '15:00:05.741",' '"error":' '{' '"time":' '"2024-05-30T15:00:19.450749187+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 450114985850568707, '"checkpoint_time":' '"2024-05-30' '15:00:05.741",' '"error":' '{' '"time":' '"2024-05-30T15:00:19.450749187+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5509.out cli changefeed remove -c changefeed-error [Thu May 30 15:00:21 CST 2024] <<<<<< START cdc server in kafka_sink_error_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_sink_error_resume.96159617.out server --log-file /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Changefeed remove successfully. ID: changefeed-error CheckpointTs: 450114985850568707 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2132f1840006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:3631, start at 2024-05-30 15:00:23.273648492 +0800 CST m=+5.868201120 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:23.286 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:23.265 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:23.265 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Thu May 30 15:00:23 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.54605462.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:00:24 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74a6790e-f75d-455c-973d-24985a2579fc {"id":"74a6790e-f75d-455c-973d-24985a2579fc","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c7d9ac2 74a6790e-f75d-455c-973d-24985a2579fc /tidb/cdc/default/default/upstream/7374683939235789593 {"id":7374683939235789593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74a6790e-f75d-455c-973d-24985a2579fc {"id":"74a6790e-f75d-455c-973d-24985a2579fc","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c7d9ac2 74a6790e-f75d-455c-973d-24985a2579fc /tidb/cdc/default/default/upstream/7374683939235789593 {"id":7374683939235789593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74a6790e-f75d-455c-973d-24985a2579fc {"id":"74a6790e-f75d-455c-973d-24985a2579fc","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c7d9ac2 74a6790e-f75d-455c-973d-24985a2579fc /tidb/cdc/default/default/upstream/7374683939235789593 {"id":7374683939235789593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:00:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {UpstreamID:7374683888318620092 Namespace:default ID:7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-8276?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:00:15.457508616 +0800 CST StartTs:450114988192038918 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00385ec60 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450114988388646919} {CheckpointTs:450114989358579723 MinTableBarrierTs:450114990669299722 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 450114989358579723, checkpointTs: 450114989358579723, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c44f0900-c272-4c67-b6ae-f21338bfb7f6 {"id":"c44f0900-c272-4c67-b6ae-f21338bfb7f6","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c531f4d c44f0900-c272-4c67-b6ae-f21338bfb7f6 /tidb/cdc/default/default/changefeed/info/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {"upstream-id":7374683888318620092,"namespace":"default","changefeed-id":"7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-8276?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:00:15.457508616+08:00","start-ts":450114988192038918,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450114988388646919} /tidb/cdc/default/default/changefeed/status/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {"checkpoint-ts":450114989358579723,"min-table-barrier-ts":450114990669299722,"admin-job-type":0} /tidb/cdc/default/default/task/position/c44f0900-c272-4c67-b6ae-f21338bfb7f6/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374683888318620092 {"id":7374683888318620092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {UpstreamID:7374683888318620092 Namespace:default ID:7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-8276?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:00:15.457508616 +0800 CST StartTs:450114988192038918 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00385ec60 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450114988388646919} {CheckpointTs:450114989358579723 MinTableBarrierTs:450114990669299722 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 450114989358579723, checkpointTs: 450114989358579723, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c44f0900-c272-4c67-b6ae-f21338bfb7f6 {"id":"c44f0900-c272-4c67-b6ae-f21338bfb7f6","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c531f4d c44f0900-c272-4c67-b6ae-f21338bfb7f6 /tidb/cdc/default/default/changefeed/info/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {"upstream-id":7374683888318620092,"namespace":"default","changefeed-id":"7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-8276?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:00:15.457508616+08:00","start-ts":450114988192038918,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450114988388646919} /tidb/cdc/default/default/changefeed/status/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {"checkpoint-ts":450114989358579723,"min-table-barrier-ts":450114990669299722,"admin-job-type":0} /tidb/cdc/default/default/task/position/c44f0900-c272-4c67-b6ae-f21338bfb7f6/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374683888318620092 {"id":7374683888318620092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {UpstreamID:7374683888318620092 Namespace:default ID:7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-8276?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:00:15.457508616 +0800 CST StartTs:450114988192038918 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00385ec60 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450114988388646919} {CheckpointTs:450114989358579723 MinTableBarrierTs:450114990669299722 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 450114989358579723, checkpointTs: 450114989358579723, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c44f0900-c272-4c67-b6ae-f21338bfb7f6 {"id":"c44f0900-c272-4c67-b6ae-f21338bfb7f6","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c531f4d c44f0900-c272-4c67-b6ae-f21338bfb7f6 /tidb/cdc/default/default/changefeed/info/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {"upstream-id":7374683888318620092,"namespace":"default","changefeed-id":"7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-8276?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:00:15.457508616+08:00","start-ts":450114988192038918,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450114988388646919} /tidb/cdc/default/default/changefeed/status/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {"checkpoint-ts":450114989358579723,"min-table-barrier-ts":450114990669299722,"admin-job-type":0} /tidb/cdc/default/default/task/position/c44f0900-c272-4c67-b6ae-f21338bfb7f6/7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374683888318620092 {"id":7374683888318620092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id [Thu May 30 15:00:25 CST 2024] <<<<<< START kafka consumer in kafka_sink_error_resume case >>>>>> table test.finish_mark not exists for 10-th check, retry later Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2132f1840006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:3631, start at 2024-05-30 15:00:23.273648492 +0800 CST m=+5.868201120 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:23.286 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:23.265 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:23.265 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2132f7000013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:3711, start at 2024-05-30 15:00:23.653870377 +0800 CST m=+6.082382651 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:23.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:23.666 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:23.666 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 3-th time... start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl_v2 Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Thu May 30 15:00:26 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.55945596.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 "id": "c44f0900-c272-4c67-b6ae-f21338bfb7f6", "cluster-id": "default" run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:00:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09877965-42b1-40d5-a5e7-12515122d817 {"id":"09877965-42b1-40d5-a5e7-12515122d817","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c7b98c4 09877965-42b1-40d5-a5e7-12515122d817 /tidb/cdc/default/default/upstream/7374683945805644991 {"id":7374683945805644991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09877965-42b1-40d5-a5e7-12515122d817 {"id":"09877965-42b1-40d5-a5e7-12515122d817","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c7b98c4 09877965-42b1-40d5-a5e7-12515122d817 /tidb/cdc/default/default/upstream/7374683945805644991 {"id":7374683945805644991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09877965-42b1-40d5-a5e7-12515122d817 {"id":"09877965-42b1-40d5-a5e7-12515122d817","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c7b98c4 09877965-42b1-40d5-a5e7-12515122d817 /tidb/cdc/default/default/upstream/7374683945805644991 {"id":7374683945805644991,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Thu May 30 15:00:26 CST 2024] <<<<<< START kafka consumer in changefeed_pause_resume case >>>>>> check_changefeed_status 127.0.0.1:8300 7736988b-32b1-4868-9cc6-ece35aeef4e0 warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=7736988b-32b1-4868-9cc6-ece35aeef4e0 + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/7736988b-32b1-4868-9cc6-ece35aeef4e0/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1070 0 --:--:-- --:--:-- --:--:-- 1075 + info='{"state":"normal","resolved_ts":450114991096070169,"checkpoint_ts":450114991096070169}' + echo '{"state":"normal","resolved_ts":450114991096070169,"checkpoint_ts":450114991096070169}' {"state":"normal","resolved_ts":450114991096070169,"checkpoint_ts":450114991096070169} ++ echo '{"state":"normal","resolved_ts":450114991096070169,"checkpoint_ts":450114991096070169}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later table test.finish_mark not exists for 11-th check, retry later Verifying downstream PD is started... table changefeed_pause_resume.t1 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.4937.out cli tso query --pd=http://127.0.0.1:2379 capture_id: c44f0900-c272-4c67-b6ae-f21338bfb7f6 check_processor_table_count http://127.0.0.1:2379 7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 c44f0900-c272-4c67-b6ae-f21338bfb7f6 1 run task successfully check_processor_table_count http://127.0.0.1:2379 7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 c44f0900-c272-4c67-b6ae-f21338bfb7f6 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later check_changefeed_status 127.0.0.1:8300 7736988b-32b1-4868-9cc6-ece35aeef4e0 warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=7736988b-32b1-4868-9cc6-ece35aeef4e0 + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/7736988b-32b1-4868-9cc6-ece35aeef4e0/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 571 0 --:--:-- --:--:-- --:--:-- 573 + info='{"state":"normal","resolved_ts":450114991803858951,"checkpoint_ts":450114991240249357}' + echo '{"state":"normal","resolved_ts":450114991803858951,"checkpoint_ts":450114991240249357}' {"state":"normal","resolved_ts":450114991803858951,"checkpoint_ts":450114991240249357} ++ echo '{"state":"normal","resolved_ts":450114991803858951,"checkpoint_ts":450114991240249357}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 2-th time, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark exists check diff successfully + set +x + tso='450114991742255105 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114991742255105 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:00:29 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.49774979.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:00:29 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/513a4a38-6e74-4fc0-81a1-89f2cc7930b4 {"id":"513a4a38-6e74-4fc0-81a1-89f2cc7930b4","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf2200a 513a4a38-6e74-4fc0-81a1-89f2cc7930b4 /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/513a4a38-6e74-4fc0-81a1-89f2cc7930b4 {"id":"513a4a38-6e74-4fc0-81a1-89f2cc7930b4","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf2200a 513a4a38-6e74-4fc0-81a1-89f2cc7930b4 /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/513a4a38-6e74-4fc0-81a1-89f2cc7930b4 {"id":"513a4a38-6e74-4fc0-81a1-89f2cc7930b4","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf2200a 513a4a38-6e74-4fc0-81a1-89f2cc7930b4 /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5640.out cli changefeed create --start-ts=450114981826658305 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-1 * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7374683796122836486,"namespace":"default","id":"changefeed-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:00:30.131517674+08:00","start_ts":450114981826658305,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114981826658305,"checkpoint_ts":450114981826658305,"checkpoint_time":"2024-05-30 14:59:50.391"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_processor_table_count http://127.0.0.1:2379 7dcaaf8a-160f-4fed-a1e6-be3fb5ff1b65 c44f0900-c272-4c67-b6ae-f21338bfb7f6 0 run task successfully ***************** properties ***************** "requestdistribution"="uniform" "updateproportion"="0" "mysql.db"="changefeed_reconstruct" "scanproportion"="0" "readproportion"="0" "mysql.user"="root" "threadcount"="4" "dotransactions"="false" "workload"="core" "operationcount"="0" "insertproportion"="0" "recordcount"="50" "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.port"="4000" ********************************************** Run finished, takes 19.097346ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3236.0, Avg(us): 1489, Min(us): 945, Max(us): 4777, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table changefeed_pause_resume.t2 not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 570 0 --:--:-- --:--:-- --:--:-- 573 + info='{"state":"normal","resolved_ts":450114981826658305,"checkpoint_ts":450114981826658305}' + echo '{"state":"normal","resolved_ts":450114981826658305,"checkpoint_ts":450114981826658305}' {"state":"normal","resolved_ts":450114981826658305,"checkpoint_ts":450114981826658305} ++ echo '{"state":"normal","resolved_ts":450114981826658305,"checkpoint_ts":450114981826658305}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: kafka... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_status 127.0.0.1:8300 7736988b-32b1-4868-9cc6-ece35aeef4e0 warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=7736988b-32b1-4868-9cc6-ece35aeef4e0 + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/7736988b-32b1-4868-9cc6-ece35aeef4e0/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 2243 0 --:--:-- --:--:-- --:--:-- 2238 100 244 100 244 0 0 2241 0 --:--:-- --:--:-- --:--:-- 2238 + info='{"state":"warning","resolved_ts":450114992852434951,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":450114992852434951,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":450114992852434951,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":450114992852434951,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":450114992852434951,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .last_warning.message + error_msg='kafka sink injected error' + [[ ! kafka sink injected error =~ kafka ]] run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:00:33 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4bdaa592-44c4-43f7-ad4c-3ff1bc5b1a8a {"id":"4bdaa592-44c4-43f7-ad4c-3ff1bc5b1a8a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c9356e3 4bdaa592-44c4-43f7-ad4c-3ff1bc5b1a8a /tidb/cdc/default/default/upstream/7374683960998465261 {"id":7374683960998465261,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4bdaa592-44c4-43f7-ad4c-3ff1bc5b1a8a {"id":"4bdaa592-44c4-43f7-ad4c-3ff1bc5b1a8a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c9356e3 4bdaa592-44c4-43f7-ad4c-3ff1bc5b1a8a /tidb/cdc/default/default/upstream/7374683960998465261 {"id":7374683960998465261,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4bdaa592-44c4-43f7-ad4c-3ff1bc5b1a8a {"id":"4bdaa592-44c4-43f7-ad4c-3ff1bc5b1a8a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84c9356e3 4bdaa592-44c4-43f7-ad4c-3ff1bc5b1a8a /tidb/cdc/default/default/upstream/7374683960998465261 {"id":7374683960998465261,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5025.out cli changefeed create --start-ts=450114991742255105 '--sink-uri=kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-26801?protocol=open-protocol&partition-num=4&kafka-client-id=ddl_puller_lag&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: fda4d388-b7aa-4f42-ba47-8f3a509fdb94 Info: {"upstream_id":7374683960998465261,"namespace":"default","id":"fda4d388-b7aa-4f42-ba47-8f3a509fdb94","sink_uri":"kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-26801?protocol=open-protocol\u0026partition-num=4\u0026kafka-client-id=ddl_puller_lag\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:00:33.487847925+08:00","start_ts":450114991742255105,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114991742255105,"checkpoint_ts":450114991742255105,"checkpoint_time":"2024-05-30 15:00:28.216"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later check_changefeed_status 127.0.0.1:8300 7736988b-32b1-4868-9cc6-ece35aeef4e0 normal + endpoint=127.0.0.1:8300 + changefeed_id=7736988b-32b1-4868-9cc6-ece35aeef4e0 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/7736988b-32b1-4868-9cc6-ece35aeef4e0/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 1869 0 --:--:-- --:--:-- --:--:-- 1876 + info='{"state":"warning","resolved_ts":450114993114578960,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":450114993114578960,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":450114993114578960,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":450114993114578960,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 252 100 252 0 0 2329 0 --:--:-- --:--:-- --:--:-- 2333 + info='{"state":"warning","resolved_ts":450114982245826579,"checkpoint_ts":450114982245826579,"last_warning":{"time":"2024-05-30T15:00:32.00686894+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":450114982245826579,"checkpoint_ts":450114982245826579,"last_warning":{"time":"2024-05-30T15:00:32.00686894+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":450114982245826579,"checkpoint_ts":450114982245826579,"last_warning":{"time":"2024-05-30T15:00:32.00686894+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":450114982245826579,"checkpoint_ts":450114982245826579,"last_warning":{"time":"2024-05-30T15:00:32.00686894+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":450114982245826579,"checkpoint_ts":450114982245826579,"last_warning":{"time":"2024-05-30T15:00:32.00686894+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .last_warning.message + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5696.out cli changefeed remove -c changefeed-error-1 + set +x [Thu May 30 15:00:34 CST 2024] <<<<<< START kafka consumer in ddl_puller_lag case >>>>>> Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 450114982245826579 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_pause_resume.t3 exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_status 127.0.0.1:8300 7736988b-32b1-4868-9cc6-ece35aeef4e0 normal + endpoint=127.0.0.1:8300 + changefeed_id=7736988b-32b1-4868-9cc6-ece35aeef4e0 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/7736988b-32b1-4868-9cc6-ece35aeef4e0/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 2286 0 --:--:-- --:--:-- --:--:-- 2301 + info='{"state":"warning","resolved_ts":450114993638866952,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":450114993638866952,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":450114993638866952,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":450114993638866952,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later + set +x wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:00:37 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Thu May 30 15:00:37 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2133b1e4001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:8228, start at 2024-05-30 15:00:35.606886776 +0800 CST m=+6.181489195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:35.615 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:35.577 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:35.577 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2133b1e4001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:8228, start at 2024-05-30 15:00:35.606886776 +0800 CST m=+6.181489195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:35.615 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:35.577 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:35.577 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2133b06c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:8308, start at 2024-05-30 15:00:35.488485729 +0800 CST m=+5.921548972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:35.497 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:35.483 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:35.483 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.57595761.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc) 3306805760 bytes in 19.16 secs (172589695 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { The 1 times to try to start tidb cluster... [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2133cfe40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:8892, start at 2024-05-30 15:00:37.544248093 +0800 CST m=+6.569356875 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:37.554 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:37.547 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:37.547 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2133cfe40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:8892, start at 2024-05-30 15:00:37.544248093 +0800 CST m=+6.569356875 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:37.554 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:37.547 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:37.547 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2133cf400008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lscxr, pid:8973, start at 2024-05-30 15:00:37.467081498 +0800 CST m=+6.382991066 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:37.475 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:37.456 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:37.456 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later [Pipeline] sh [Pipeline] sh [Pipeline] sh [Thu May 30 15:00:39 CST 2024] <<<<<< START cdc server in multi_tables_ddl_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl_v2.95469548.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 check_changefeed_status 127.0.0.1:8300 7736988b-32b1-4868-9cc6-ece35aeef4e0 normal + endpoint=127.0.0.1:8300 + changefeed_id=7736988b-32b1-4868-9cc6-ece35aeef4e0 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/7736988b-32b1-4868-9cc6-ece35aeef4e0/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 2247 0 --:--:-- --:--:-- --:--:-- 2259 + info='{"state":"warning","resolved_ts":450114994700550146,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":450114994700550146,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":450114994700550146,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":450114994700550146,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 3-th time, retry later + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:00:40 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/050958fe-696f-44ea-82ae-c9ca7118ed1e {"id":"050958fe-696f-44ea-82ae-c9ca7118ed1e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf2207d 050958fe-696f-44ea-82ae-c9ca7118ed1e /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/050958fe-696f-44ea-82ae-c9ca7118ed1e {"id":"050958fe-696f-44ea-82ae-c9ca7118ed1e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf2207d 050958fe-696f-44ea-82ae-c9ca7118ed1e /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/050958fe-696f-44ea-82ae-c9ca7118ed1e {"id":"050958fe-696f-44ea-82ae-c9ca7118ed1e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf2207d 050958fe-696f-44ea-82ae-c9ca7118ed1e /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5802.out cli changefeed create --start-ts=450114981826658305 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-2 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7374683796122836486,"namespace":"default","id":"changefeed-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:00:41.397485215+08:00","start_ts":450114981826658305,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114981826658305,"checkpoint_ts":450114981826658305,"checkpoint_time":"2024-05-30 14:59:50.391"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10198.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 3-th time, retry later table test.finish_mark not exists for 1-th check, retry later + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 table test.finish_mark not exists for 2-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-30T15:00:41.407223146+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450114981826658305 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-30T15:00:41.407223146+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450114981826658305 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-30T15:00:41.407223146+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450114981826658305 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-05-30T15:00:41.407223146+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450114981826658305 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-05-30T15:00:41.407223146+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450114981826658305 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450114981826658305 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450114981826658305 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5865.out cli changefeed remove -c changefeed-error-2 + set +x + tso='450114995289325569 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450114995289325569 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:00:43 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1023510237.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:00:43 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5234a06-2dd1-464d-bb5f-0ebe38ccce96 {"id":"a5234a06-2dd1-464d-bb5f-0ebe38ccce96","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84cc65cc0 a5234a06-2dd1-464d-bb5f-0ebe38ccce96 /tidb/cdc/default/default/upstream/7374684015600990862 {"id":7374684015600990862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5234a06-2dd1-464d-bb5f-0ebe38ccce96 {"id":"a5234a06-2dd1-464d-bb5f-0ebe38ccce96","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84cc65cc0 a5234a06-2dd1-464d-bb5f-0ebe38ccce96 /tidb/cdc/default/default/upstream/7374684015600990862 {"id":7374684015600990862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5234a06-2dd1-464d-bb5f-0ebe38ccce96 {"id":"a5234a06-2dd1-464d-bb5f-0ebe38ccce96","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84cc65cc0 a5234a06-2dd1-464d-bb5f-0ebe38ccce96 /tidb/cdc/default/default/upstream/7374684015600990862 {"id":7374684015600990862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7374684015600990862,"namespace":"default","id":"test-normal","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-normal-17022?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:00:43.31173082+08:00","start_ts":450114994811961345,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114994811961345,"checkpoint_ts":450114994811961345,"checkpoint_time":"2024-05-30 15:00:39.926"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7374684015600990862,"namespace":"default","id":"test-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-1-2717?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:00:43.546172696+08:00","start_ts":450114994811961345,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114994811961345,"checkpoint_ts":450114994811961345,"checkpoint_time":"2024-05-30 15:00:39.926"} check diff failed 4-th time, retry later Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7374684015600990862,"namespace":"default","id":"test-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-2-19935?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:00:43.744741709+08:00","start_ts":450114994811961345,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114994811961345,"checkpoint_ts":450114994811961345,"checkpoint_time":"2024-05-30 15:00:39.926"} [Thu May 30 15:00:43 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Thu May 30 15:00:43 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Thu May 30 15:00:43 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 0 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table test.finish_mark not exists for 3-th check, retry later + set +x check diff failed 5-th time, retry later Verifying downstream PD is started... wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:00:46 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81cc1941-b99c-4737-bdb1-b67d43e26901 {"id":"81cc1941-b99c-4737-bdb1-b67d43e26901","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84cc44cc6 81cc1941-b99c-4737-bdb1-b67d43e26901 /tidb/cdc/default/default/upstream/7374684018142408324 {"id":7374684018142408324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81cc1941-b99c-4737-bdb1-b67d43e26901 {"id":"81cc1941-b99c-4737-bdb1-b67d43e26901","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84cc44cc6 81cc1941-b99c-4737-bdb1-b67d43e26901 /tidb/cdc/default/default/upstream/7374684018142408324 {"id":7374684018142408324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81cc1941-b99c-4737-bdb1-b67d43e26901 {"id":"81cc1941-b99c-4737-bdb1-b67d43e26901","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84cc44cc6 81cc1941-b99c-4737-bdb1-b67d43e26901 /tidb/cdc/default/default/upstream/7374684018142408324 {"id":7374684018142408324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10283.out cli changefeed create --start-ts=450114995289325569 '--sink-uri=kafka://127.0.0.1:9092/ticdc-partition-table-test-7377?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 4-th check, retry later Create changefeed successfully! ID: 5584c946-7429-4396-9e74-1c564bbe9987 Info: {"upstream_id":7374684018142408324,"namespace":"default","id":"5584c946-7429-4396-9e74-1c564bbe9987","sink_uri":"kafka://127.0.0.1:9092/ticdc-partition-table-test-7377?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:00:46.848074069+08:00","start_ts":450114995289325569,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114995289325569,"checkpoint_ts":450114995289325569,"checkpoint_time":"2024-05-30 15:00:41.747"} PASS cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Thu May 30 15:00:46 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.59105912.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_changefeed_status 127.0.0.1:8300 7736988b-32b1-4868-9cc6-ece35aeef4e0 normal + endpoint=127.0.0.1:8300 + changefeed_id=7736988b-32b1-4868-9cc6-ece35aeef4e0 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/7736988b-32b1-4868-9cc6-ece35aeef4e0/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 2263 0 --:--:-- --:--:-- --:--:-- 2259 100 244 100 244 0 0 2260 0 --:--:-- --:--:-- --:--:-- 2259 + info='{"state":"warning","resolved_ts":450114996273414148,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":450114996273414148,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":450114996273414148,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":450114996273414148,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 4-th time, retry later table multi_tables_ddl_test.t55 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 6-th time, retry later + set +x [Thu May 30 15:00:48 CST 2024] <<<<<< START kafka consumer in partition_table case >>>>>> table test.finish_mark not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 not exists for 2-th check, retry later check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:00:50 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3ab4a537-eccf-4121-95ee-58971b12c66d {"id":"3ab4a537-eccf-4121-95ee-58971b12c66d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf220d6 3ab4a537-eccf-4121-95ee-58971b12c66d /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3ab4a537-eccf-4121-95ee-58971b12c66d {"id":"3ab4a537-eccf-4121-95ee-58971b12c66d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf220d6 3ab4a537-eccf-4121-95ee-58971b12c66d /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3ab4a537-eccf-4121-95ee-58971b12c66d {"id":"3ab4a537-eccf-4121-95ee-58971b12c66d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318fc84bf220d6 3ab4a537-eccf-4121-95ee-58971b12c66d /tidb/cdc/default/default/upstream/7374683796122836486 {"id":7374683796122836486,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5952.out cli changefeed create --start-ts=0 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-initialize-error table test.finish_mark exists check diff successfully Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7374683796122836486,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:00:50.492134458+08:00","start_ts":450114997541928965,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450114997541928965,"checkpoint_ts":450114997541928965,"checkpoint_time":"2024-05-30 15:00:50.340"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table multi_tables_ddl_test.t55 not exists for 3-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s check diff failed 1-th time, retry later wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450114997541928965, "checkpoint_time": "2024-05-30 15:00:50.340", "error": { "time": "2024-05-30T15:00:50.663968824+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450114997541928965, "checkpoint_time": "2024-05-30 15:00:50.340", "error": { "time": "2024-05-30T15:00:50.663968824+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450114997541928965, "checkpoint_time": "2024-05-30 15:00:50.340", "error": { "time": "2024-05-30T15:00:50.663968824+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 450114997541928965, '"checkpoint_time":' '"2024-05-30' '15:00:50.340",' '"error":' '{' '"time":' '"2024-05-30T15:00:50.663968824+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:00:52 CST 2024] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table multi_tables_ddl_test.t55 not exists for 4-th check, retry later check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450114998446587907, "checkpoint_time": "2024-05-30 15:00:53.791", "error": null }' + echo '{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450114998446587907, "checkpoint_time": "2024-05-30 15:00:53.791", "error": null }' { "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450114998446587907, "checkpoint_time": "2024-05-30 15:00:53.791", "error": null } ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450114998446587907, '"checkpoint_time":' '"2024-05-30' '15:00:53.791",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450114998446587907, '"checkpoint_time":' '"2024-05-30' '15:00:53.791",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6057.out cli changefeed pause -c changefeed-initialize-error PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... check_changefeed_status 127.0.0.1:8300 7736988b-32b1-4868-9cc6-ece35aeef4e0 normal + endpoint=127.0.0.1:8300 + changefeed_id=7736988b-32b1-4868-9cc6-ece35aeef4e0 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/7736988b-32b1-4868-9cc6-ece35aeef4e0/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 2233 0 --:--:-- --:--:-- --:--:-- 2238 + info='{"state":"warning","resolved_ts":450114998370566148,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":450114998370566148,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":450114998370566148,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":450114998370566148,"checkpoint_ts":450114991240249357,"last_warning":{"time":"2024-05-30T15:00:29.134402418+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 5-th time, retry later table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2134e9900010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k, pid:3988, start at 2024-05-30 15:00:55.545047123 +0800 CST m=+5.884763653 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:55.554 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:55.524 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:55.524 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450114998446587907, "checkpoint_time": "2024-05-30 15:00:53.791", "error": { "time": "2024-05-30T15:00:50.663968824+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450114998446587907, "checkpoint_time": "2024-05-30 15:00:53.791", "error": { "time": "2024-05-30T15:00:50.663968824+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450114998446587907, "checkpoint_time": "2024-05-30 15:00:53.791", "error": { "time": "2024-05-30T15:00:50.663968824+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 450114998446587907, '"checkpoint_time":' '"2024-05-30' '15:00:53.791",' '"error":' '{' '"time":' '"2024-05-30T15:00:50.663968824+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 450114998446587907, '"checkpoint_time":' '"2024-05-30' '15:00:53.791",' '"error":' '{' '"time":' '"2024-05-30T15:00:50.663968824+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6110.out cli changefeed resume -c changefeed-initialize-error PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2134e9900010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k, pid:3988, start at 2024-05-30 15:00:55.545047123 +0800 CST m=+5.884763653 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:55.554 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:55.524 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:55.524 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2134ebd4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k, pid:4060, start at 2024-05-30 15:00:55.686213434 +0800 CST m=+5.931648099 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:02:55.692 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:00:55.669 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:50:55.669 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 4-th time, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450114998446587907, "checkpoint_time": "2024-05-30 15:00:53.791", "error": null }' + echo '{ "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450114998446587907, "checkpoint_time": "2024-05-30 15:00:53.791", "error": null }' { "upstream_id": 7374683796122836486, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450114998446587907, "checkpoint_time": "2024-05-30 15:00:53.791", "error": null } ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450114998446587907, '"checkpoint_time":' '"2024-05-30' '15:00:53.791",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7374683796122836486, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450114998446587907, '"checkpoint_time":' '"2024-05-30' '15:00:53.791",' '"error":' null '}' + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6179.out cli changefeed remove -c changefeed-initialize-error table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 450114998446587907 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-11776?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + info='{ "upstream_id": 7374684015600990862, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450114996686290972, "checkpoint_time": "2024-05-30 15:00:47.076", "error": null }' + echo '{ "upstream_id": 7374684015600990862, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450114996686290972, "checkpoint_time": "2024-05-30 15:00:47.076", "error": null }' { "upstream_id": 7374684015600990862, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450114996686290972, "checkpoint_time": "2024-05-30 15:00:47.076", "error": null } ++ echo '{' '"upstream_id":' 7374684015600990862, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450114996686290972, '"checkpoint_time":' '"2024-05-30' '15:00:47.076",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374684015600990862, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450114996686290972, '"checkpoint_time":' '"2024-05-30' '15:00:47.076",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7374684015600990862, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450114999700946953, "checkpoint_time": "2024-05-30 15:00:58.576", "error": null }' + echo '{ "upstream_id": 7374684015600990862, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450114999700946953, "checkpoint_time": "2024-05-30 15:00:58.576", "error": null }' { "upstream_id": 7374684015600990862, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450114999700946953, "checkpoint_time": "2024-05-30 15:00:58.576", "error": null } ++ echo '{' '"upstream_id":' 7374684015600990862, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450114999700946953, '"checkpoint_time":' '"2024-05-30' '15:00:58.576",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374684015600990862, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450114999700946953, '"checkpoint_time":' '"2024-05-30' '15:00:58.576",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7374684015600990862, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450114996175110182, "checkpoint_time": "2024-05-30 15:00:45.126", "error": { "time": "2024-05-30T15:00:47.629695027+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7374684015600990862, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450114996175110182, "checkpoint_time": "2024-05-30 15:00:45.126", "error": { "time": "2024-05-30T15:00:47.629695027+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7374684015600990862, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450114996175110182, "checkpoint_time": "2024-05-30 15:00:45.126", "error": { "time": "2024-05-30T15:00:47.629695027+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7374684015600990862, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450114996175110182, '"checkpoint_time":' '"2024-05-30' '15:00:45.126",' '"error":' '{' '"time":' '"2024-05-30T15:00:47.629695027+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7374684015600990862, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450114996175110182, '"checkpoint_time":' '"2024-05-30' '15:00:45.126",' '"error":' '{' '"time":' '"2024-05-30T15:00:47.629695027+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' ++ jq -r .error.message + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully check diff successfully [Thu May 30 15:01:00 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.52905292.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... + set +x wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:01:02 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:01:02 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [Pipeline] } < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:03 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c2b1989a-7821-4c6d-a42b-6f2b22de0f9e {"id":"c2b1989a-7821-4c6d-a42b-6f2b22de0f9e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d0d74bf c2b1989a-7821-4c6d-a42b-6f2b22de0f9e /tidb/cdc/default/default/upstream/7374684095027398009 {"id":7374684095027398009,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c2b1989a-7821-4c6d-a42b-6f2b22de0f9e {"id":"c2b1989a-7821-4c6d-a42b-6f2b22de0f9e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d0d74bf c2b1989a-7821-4c6d-a42b-6f2b22de0f9e /tidb/cdc/default/default/upstream/7374684095027398009 {"id":7374684095027398009,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c2b1989a-7821-4c6d-a42b-6f2b22de0f9e {"id":"c2b1989a-7821-4c6d-a42b-6f2b22de0f9e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d0d74bf c2b1989a-7821-4c6d-a42b-6f2b22de0f9e /tidb/cdc/default/default/upstream/7374684095027398009 {"id":7374684095027398009,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5334.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cdc-test-17981?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } Create changefeed successfully! ID: 771c43f8-807f-4cf3-bf23-bc8aea728591 Info: {"upstream_id":7374684095027398009,"namespace":"default","id":"771c43f8-807f-4cf3-bf23-bc8aea728591","sink_uri":"kafka://127.0.0.1:9092/ticdc-cdc-test-17981?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:01:03.687335597+08:00","start_ts":450115000998035459,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115000998035459,"checkpoint_ts":450115000998035459,"checkpoint_time":"2024-05-30 15:01:03.524"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout check diff failed 2-th time, retry later [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] sh check_changefeed_status 127.0.0.1:8300 7736988b-32b1-4868-9cc6-ece35aeef4e0 normal + endpoint=127.0.0.1:8300 + changefeed_id=7736988b-32b1-4868-9cc6-ece35aeef4e0 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/7736988b-32b1-4868-9cc6-ece35aeef4e0/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 799 0 --:--:-- --:--:-- --:--:-- 803 + info='{"state":"normal","resolved_ts":450115001254150150,"checkpoint_ts":450115000992006152}' + echo '{"state":"normal","resolved_ts":450115001254150150,"checkpoint_ts":450115000992006152}' {"state":"normal","resolved_ts":450115001254150150,"checkpoint_ts":450115000992006152} ++ echo '{"state":"normal","resolved_ts":450115001254150150,"checkpoint_ts":450115000992006152}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z '' ]] ++ echo '{"state":"normal","resolved_ts":450115001254150150,"checkpoint_ts":450115000992006152}' ++ jq -r .last_error + error_msg=null + [[ ! null == \n\u\l\l ]] ++ echo '{"state":"normal","resolved_ts":450115001254150150,"checkpoint_ts":450115000992006152}' ++ jq -r .last_warning + error_msg=null + [[ ! null == \n\u\l\l ]] + exit 0 run task successfully table kafka_sink_error_resume.t1 exists table kafka_sink_error_resume.t2 exists check diff successfully check diff failed 1-th time, retry later [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=8aaf7b07-3fdb-4c58-bca0-dac79cd3d756 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-3v2gz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-3v2gz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2135695c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t, pid:6451, start at 2024-05-30 15:01:03.706217491 +0800 CST m=+6.176454802 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:03.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:03.703 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:03.703 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2135695c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t, pid:6451, start at 2024-05-30 15:01:03.706217491 +0800 CST m=+6.176454802 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:03.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:03.703 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:03.703 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21356a440014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-1ng0t, pid:6506, start at 2024-05-30 15:01:03.802893109 +0800 CST m=+6.161346259 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:03.809 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:03.811 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:03.811 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure [Pipeline] sh Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=1e3997ba-c15f-469b-9d94-3599b3fc0e27 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-nhz9r GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3 ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-nhz9r GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G16 [Pipeline] sh check diff failed 3-th time, retry later Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=e5b00b22-bf56-4560-bd66-c55295909a8a BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-j67b2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-2gdb1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-j67b2 ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-2gdb1 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-j67b2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:01:06 CST 2024] <<<<<< run test case owner_resign success! >>>>>> + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G13 [Pipeline] sh Run cases: tiflash region_merge common_1 PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=0d52c8da-c436-47ce-9cd7-e330e65a370b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ld79s GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ld79s GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G14 Run cases: big_txn changefeed_finish force_replicate_table PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=6c377e9a-4dda-4222-af83-4f55cf79eb12 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ls1gk GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-ls1gk GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/big_txn/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff successfully + set +x [Thu May 30 15:01:07 CST 2024] <<<<<< START kafka consumer in cdc case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240527072219-a97e6464c01d go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240527072219-a97e6464c01d go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/multierr v1.11.0 wait process cdc.test exit for 1-th time... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G17 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.7712.out cli tso query --pd=http://127.0.0.1:2379 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=38b71a34-4ef2-4284-8fe6-97d5e43c4abb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-mjlz0 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-5rms5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-5rms5 ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-mjlz0 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-mjlz0 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:01:07 CST 2024] <<<<<< skip test case clustered_index for kafka! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:01:07 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_handle_key_only Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... check diff failed 4-th time, retry later wait process cdc.test exit for 3-th time... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=c3563727-a6b9-4e56-ab1a-cf89588cca6d JENKINS_NODE_COOKIE=d9e103e7-22f7-4530-a3d6-788a7db285ca BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1796069089731416064","prowjobid":"c3563727-a6b9-4e56-ab1a-cf89588cca6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#727 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796069089731416064 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/727/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-pz6n0 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_727-pz6n0 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=727 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: kafka... <<================= cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:01:09 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... + set +x + tso='450115002263928833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115002263928833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:01:10 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> ***************** properties ***************** "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "recordcount"="10" "dotransactions"="false" "workload"="core" "insertproportion"="0" "mysql.db"="multi_capture_1" "mysql.port"="4000" "readproportion"="0" "requestdistribution"="uniform" "threadcount"="2" "updateproportion"="0" "readallfields"="true" "operationcount"="0" ********************************************** Run finished, takes 10.252317ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1853.6, Avg(us): 1932, Min(us): 1016, Max(us): 4794, 95th(us): 5000, 99th(us): 5000 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/text v0.15.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/kvproto v0.0.0-20240522024016-df42997c2c57 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading golang.org/x/sync v0.7.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240510070134-e7f530c64790 go: downloading github.com/pingcap/tipb v0.0.0-20240507090649-2bf6bb0cb996 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/prometheus/client_golang v1.19.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/tikv/pd/client v0.0.0-20240210135946-3488a653ddd9 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading golang.org/x/sys v0.20.0 go: downloading google.golang.org/grpc v1.64.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.6 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 ***************** properties ***************** "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "readallfields"="true" "updateproportion"="0" "insertproportion"="0" "scanproportion"="0" "mysql.port"="4000" "mysql.db"="multi_capture_2" "readproportion"="0" "recordcount"="10" "operationcount"="0" "dotransactions"="false" "threadcount"="2" "workload"="core" "mysql.user"="root" ********************************************** Run finished, takes 13.971122ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1292.7, Avg(us): 2693, Min(us): 1552, Max(us): 6198, 95th(us): 7000, 99th(us): 7000 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading google.golang.org/protobuf v1.34.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/google/btree v1.1.2 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/pkg/errors v0.9.1 go: downloading golang.org/x/tools v0.21.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/api/v3 v3.5.12 check diff failed 5-th time, retry later ***************** properties ***************** "readallfields"="true" "mysql.port"="4000" "requestdistribution"="uniform" "workload"="core" "updateproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "insertproportion"="0" "dotransactions"="false" "readproportion"="0" "scanproportion"="0" "threadcount"="2" "mysql.db"="multi_capture_3" "mysql.user"="root" "recordcount"="10" ********************************************** Run finished, takes 11.46029ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1659.9, Avg(us): 2192, Min(us): 1312, Max(us): 5333, 95th(us): 6000, 99th(us): 6000 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading golang.org/x/net v0.25.0 ***************** properties ***************** "readproportion"="0" "workload"="core" "scanproportion"="0" "requestdistribution"="uniform" "updateproportion"="0" "recordcount"="10" "insertproportion"="0" "readallfields"="true" "threadcount"="2" "mysql.port"="4000" "mysql.host"="127.0.0.1" "mysql.db"="multi_capture_4" "operationcount"="0" "mysql.user"="root" "dotransactions"="false" ********************************************** Run finished, takes 15.546924ms INSERT - Takes(s): 0.0, Count: 9, OPS: 1025.3, Avg(us): 2572, Min(us): 1698, Max(us): 6662, 95th(us): 7000, 99th(us): 7000 [Thu May 30 15:01:10 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.782378257827.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.15.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/sasha-s/go-deadlock v0.3.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/dolthub/maphash v0.1.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 [Pipeline] // cache [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... [Pipeline] // dir [Pipeline] } go: downloading github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/DataDog/zstd v1.5.5 [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 [Pipeline] // withEnv check diff successfully [Pipeline] } The 1 times to try to start tidb cluster... [Pipeline] // node [Pipeline] } find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:01:13 CST 2024] <<<<<< run test case sink_hang success! >>>>>> [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:13 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6ed2d0d-c018-477f-96ec-c9da894482d5 {"id":"d6ed2d0d-c018-477f-96ec-c9da894482d5","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e302e d6ed2d0d-c018-477f-96ec-c9da894482d5 /tidb/cdc/default/default/upstream/7374684138235026359 {"id":7374684138235026359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6ed2d0d-c018-477f-96ec-c9da894482d5 {"id":"d6ed2d0d-c018-477f-96ec-c9da894482d5","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e302e d6ed2d0d-c018-477f-96ec-c9da894482d5 /tidb/cdc/default/default/upstream/7374684138235026359 {"id":7374684138235026359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6ed2d0d-c018-477f-96ec-c9da894482d5 {"id":"d6ed2d0d-c018-477f-96ec-c9da894482d5","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e302e d6ed2d0d-c018-477f-96ec-c9da894482d5 /tidb/cdc/default/default/upstream/7374684138235026359 {"id":7374684138235026359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Thu May 30 15:01:13 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.78607862.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... table partition_table.t exists table partition_table.t1 exists table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 2-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 start tidb cluster in /tmp/tidb_cdc_test/multi_topics_v2 Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:17 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bad55e21-419c-428a-97af-9a232821e8e0 {"id":"bad55e21-419c-428a-97af-9a232821e8e0","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/d6ed2d0d-c018-477f-96ec-c9da894482d5 {"id":"d6ed2d0d-c018-477f-96ec-c9da894482d5","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e302e d6ed2d0d-c018-477f-96ec-c9da894482d5 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e305c bad55e21-419c-428a-97af-9a232821e8e0 /tidb/cdc/default/default/upstream/7374684138235026359 {"id":7374684138235026359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bad55e21-419c-428a-97af-9a232821e8e0 {"id":"bad55e21-419c-428a-97af-9a232821e8e0","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/d6ed2d0d-c018-477f-96ec-c9da894482d5 {"id":"d6ed2d0d-c018-477f-96ec-c9da894482d5","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e302e d6ed2d0d-c018-477f-96ec-c9da894482d5 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e305c bad55e21-419c-428a-97af-9a232821e8e0 /tidb/cdc/default/default/upstream/7374684138235026359 {"id":7374684138235026359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bad55e21-419c-428a-97af-9a232821e8e0 {"id":"bad55e21-419c-428a-97af-9a232821e8e0","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/d6ed2d0d-c018-477f-96ec-c9da894482d5 {"id":"d6ed2d0d-c018-477f-96ec-c9da894482d5","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e302e d6ed2d0d-c018-477f-96ec-c9da894482d5 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e305c bad55e21-419c-428a-97af-9a232821e8e0 /tidb/cdc/default/default/upstream/7374684138235026359 {"id":7374684138235026359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Thu May 30 15:01:17 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.79027904.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark exists check diff successfully check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:20 GMT < Content-Length: 1144 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8cf4a698-cb86-4c15-b86e-d4cfbad92093 {"id":"8cf4a698-cb86-4c15-b86e-d4cfbad92093","address":"127.0.0.1:8303","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/bad55e21-419c-428a-97af-9a232821e8e0 {"id":"bad55e21-419c-428a-97af-9a232821e8e0","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/d6ed2d0d-c018-477f-96ec-c9da894482d5 {"id":"d6ed2d0d-c018-477f-96ec-c9da894482d5","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e302e d6ed2d0d-c018-477f-96ec-c9da894482d5 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e305c bad55e21-419c-428a-97af-9a232821e8e0 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e3076 8cf4a698-cb86-4c15-b86e-d4cfbad92093 /tidb/cdc/default/default/upstream/7374684138235026359 {"id":7374684138235026359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8cf4a698-cb86-4c15-b86e-d4cfbad92093 {"id":"8cf4a698-cb86-4c15-b86e-d4cfbad92093","address":"127.0.0.1:8303","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/bad55e21-419c-428a-97af-9a232821e8e0 {"id":"bad55e21-419c-428a-97af-9a232821e8e0","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/d6ed2d0d-c018-477f-96ec-c9da894482d5 {"id":"d6ed2d0d-c018-477f-96ec-c9da894482d5","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e302e d6ed2d0d-c018-477f-96ec-c9da894482d5 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e305c bad55e21-419c-428a-97af-9a232821e8e0 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e3076 8cf4a698-cb86-4c15-b86e-d4cfbad92093 /tidb/cdc/default/default/upstream/7374684138235026359 {"id":7374684138235026359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8cf4a698-cb86-4c15-b86e-d4cfbad92093 {"id":"8cf4a698-cb86-4c15-b86e-d4cfbad92093","address":"127.0.0.1:8303","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/bad55e21-419c-428a-97af-9a232821e8e0 {"id":"bad55e21-419c-428a-97af-9a232821e8e0","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/d6ed2d0d-c018-477f-96ec-c9da894482d5 {"id":"d6ed2d0d-c018-477f-96ec-c9da894482d5","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e302e d6ed2d0d-c018-477f-96ec-c9da894482d5 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e305c bad55e21-419c-428a-97af-9a232821e8e0 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d2e3076 8cf4a698-cb86-4c15-b86e-d4cfbad92093 /tidb/cdc/default/default/upstream/7374684138235026359 {"id":7374684138235026359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.7955.out cli changefeed create --start-ts=450115002263928833 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-capture-test-12039?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --server=127.0.0.1:8301 Create changefeed successfully! ID: 4a793491-ba56-4f5a-a00a-39f9e8eb1010 Info: {"upstream_id":7374684138235026359,"namespace":"default","id":"4a793491-ba56-4f5a-a00a-39f9e8eb1010","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-capture-test-12039?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:01:20.592492127+08:00","start_ts":450115002263928833,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115002263928833,"checkpoint_ts":450115002263928833,"checkpoint_time":"2024-05-30 15:01:08.353"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... check diff failed 3-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:01:21 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... process cdc.test already exit [Thu May 30 15:01:21 CST 2024] <<<<<< run test case partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Thu May 30 15:01:22 CST 2024] <<<<<< START kafka consumer in multi_capture case >>>>>> table multi_capture_1.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21366e600012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:6630, start at 2024-05-30 15:01:20.446714112 +0800 CST m=+8.453228436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:20.455 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:20.459 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:20.459 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21366e600012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:6630, start at 2024-05-30 15:01:20.446714112 +0800 CST m=+8.453228436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:20.455 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:20.459 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:20.459 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213670140009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:6688, start at 2024-05-30 15:01:20.526610105 +0800 CST m=+8.425395395 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:20.535 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:20.517 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:20.517 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21366970000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3, pid:1199, start at 2024-05-30 15:01:20.11144363 +0800 CST m=+5.913748175 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:20.123 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:20.092 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:20.092 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21366970000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3, pid:1199, start at 2024-05-30 15:01:20.11144363 +0800 CST m=+5.913748175 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:20.123 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:20.092 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:20.092 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21366bb40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3, pid:1281, start at 2024-05-30 15:01:20.283913903 +0800 CST m=+5.921412619 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:20.295 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:20.287 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:20.287 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213696140017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf, pid:1080, start at 2024-05-30 15:01:22.997641542 +0800 CST m=+5.656834467 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:23.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:22.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:22.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21369ea40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn, pid:1134, start at 2024-05-30 15:01:23.544879779 +0800 CST m=+5.713815875 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:23.552 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:23.547 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:23.547 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_capture_1.usertable not exists for 2-th check, retry later [Thu May 30 15:01:24 CST 2024] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.25232525.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2136ace0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr, pid:1097, start at 2024-05-30 15:01:24.429462118 +0800 CST m=+5.486678522 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:24.436 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:24.408 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:24.408 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Thu May 30 15:01:24 CST 2024] <<<<<< START cdc server in kafka_simple_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.79267928.out server --log-file /tmp/tidb_cdc_test/kafka_simple_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_handle_key_only/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213696140017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf, pid:1080, start at 2024-05-30 15:01:22.997641542 +0800 CST m=+5.656834467 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:23.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:22.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:22.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2136987c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf, pid:1147, start at 2024-05-30 15:01:23.110712586 +0800 CST m=+5.669816138 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:23.122 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:23.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:23.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/big_txn/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/big_txn/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21369ea40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn, pid:1134, start at 2024-05-30 15:01:23.544879779 +0800 CST m=+5.713815875 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:23.552 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:23.547 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:23.547 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2136a0280013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn, pid:1210, start at 2024-05-30 15:01:23.623481336 +0800 CST m=+5.688841302 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:23.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:23.594 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:23.594 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table multi_capture_1.usertable exists table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2136ace0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr, pid:1097, start at 2024-05-30 15:01:24.429462118 +0800 CST m=+5.486678522 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:24.436 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:24.408 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:24.408 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2136ae940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr, pid:1176, start at 2024-05-30 15:01:24.556975451 +0800 CST m=+5.524706807 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:24.563 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:24.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:24.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 6-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:27 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a29bc652-e347-46e4-b135-f609c94c57ca {"id":"a29bc652-e347-46e4-b135-f609c94c57ca","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d7153bd a29bc652-e347-46e4-b135-f609c94c57ca /tidb/cdc/default/default/upstream/7374684212048260929 {"id":7374684212048260929,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a29bc652-e347-46e4-b135-f609c94c57ca {"id":"a29bc652-e347-46e4-b135-f609c94c57ca","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d7153bd a29bc652-e347-46e4-b135-f609c94c57ca /tidb/cdc/default/default/upstream/7374684212048260929 {"id":7374684212048260929,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a29bc652-e347-46e4-b135-f609c94c57ca {"id":"a29bc652-e347-46e4-b135-f609c94c57ca","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d7153bd a29bc652-e347-46e4-b135-f609c94c57ca /tidb/cdc/default/default/upstream/7374684212048260929 {"id":7374684212048260929,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Thu May 30 15:01:27 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.24722474.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/mq_sink_dispatcher Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2136e1b40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:10951, start at 2024-05-30 15:01:27.819543608 +0800 CST m=+6.054386478 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:27.827 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:27.789 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:27.789 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 839965ba-60a4-40f7-9450-02851112f192 Info: {"upstream_id":7374684212048260929,"namespace":"default","id":"839965ba-60a4-40f7-9450-02851112f192","sink_uri":"kafka://127.0.0.1:9092/ticdc-new_ci_collation-test-3507?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:01:28.173024215+08:00","start_ts":450115006560206852,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115006560206852,"checkpoint_ts":450115006560206852,"checkpoint_time":"2024-05-30 15:01:24.742"} [Thu May 30 15:01:28 CST 2024] <<<<<< START kafka consumer in new_ci_collation case >>>>>> + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2489.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:28 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbd73814-6667-480e-b06c-32d346221740 {"id":"cbd73814-6667-480e-b06c-32d346221740","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d6a8ccc cbd73814-6667-480e-b06c-32d346221740 /tidb/cdc/default/default/upstream/7374684198568163077 {"id":7374684198568163077,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbd73814-6667-480e-b06c-32d346221740 {"id":"cbd73814-6667-480e-b06c-32d346221740","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d6a8ccc cbd73814-6667-480e-b06c-32d346221740 /tidb/cdc/default/default/upstream/7374684198568163077 {"id":7374684198568163077,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbd73814-6667-480e-b06c-32d346221740 {"id":"cbd73814-6667-480e-b06c-32d346221740","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d6a8ccc cbd73814-6667-480e-b06c-32d346221740 /tidb/cdc/default/default/upstream/7374684198568163077 {"id":7374684198568163077,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.7977.out cli tso query --pd=http://127.0.0.1:2379 table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2136c7bc0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l, pid:1266, start at 2024-05-30 15:01:26.147780732 +0800 CST m=+5.615932858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:26.153 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:26.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:26.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2136c7bc0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l, pid:1266, start at 2024-05-30 15:01:26.147780732 +0800 CST m=+5.615932858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:26.153 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:26.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:26.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2136c93c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l, pid:1339, start at 2024-05-30 15:01:26.231406971 +0800 CST m=+5.595939785 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:26.237 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:26.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:26.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully [Thu May 30 15:01:29 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.25412543.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='450115007534333954 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115007534333954 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.8006.out cli changefeed create --start-ts=450115007534333954 '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-11225?protocol=simple' -c simple-handle-key-only --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/conf/changefeed.toml Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2136e1b40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:10951, start at 2024-05-30 15:01:27.819543608 +0800 CST m=+6.054386478 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:27.827 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:27.789 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:27.789 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2136e3380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-9m6pz, pid:11036, start at 2024-05-30 15:01:27.924000627 +0800 CST m=+6.042179727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:27.932 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:27.935 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:27.935 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='450115007596986369 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115007596986369 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:01:30 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.25082510.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_capture_4.usertable exists check diff failed 1-th time, retry later Create changefeed successfully! ID: simple-handle-key-only Info: {"upstream_id":7374684198568163077,"namespace":"default","id":"simple-handle-key-only","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-11225?protocol=simple","create_time":"2024-05-30T15:01:30.552619862+08:00","start_ts":450115007534333954,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115007534333954,"checkpoint_ts":450115007534333954,"checkpoint_time":"2024-05-30 15:01:28.458"} PASS Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:30 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/758996db-3613-4805-b6ec-f34b16c3728e {"id":"758996db-3613-4805-b6ec-f34b16c3728e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d798ec1 758996db-3613-4805-b6ec-f34b16c3728e /tidb/cdc/default/default/upstream/7374684219425185212 {"id":7374684219425185212,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/758996db-3613-4805-b6ec-f34b16c3728e {"id":"758996db-3613-4805-b6ec-f34b16c3728e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d798ec1 758996db-3613-4805-b6ec-f34b16c3728e /tidb/cdc/default/default/upstream/7374684219425185212 {"id":7374684219425185212,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/758996db-3613-4805-b6ec-f34b16c3728e {"id":"758996db-3613-4805-b6ec-f34b16c3728e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d798ec1 758996db-3613-4805-b6ec-f34b16c3728e /tidb/cdc/default/default/upstream/7374684219425185212 {"id":7374684219425185212,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Thu May 30 15:01:30 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.25142516.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:31 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/758996db-3613-4805-b6ec-f34b16c3728e {"id":"758996db-3613-4805-b6ec-f34b16c3728e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d798ec1 758996db-3613-4805-b6ec-f34b16c3728e /tidb/cdc/default/default/upstream/7374684219425185212 {"id":7374684219425185212,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/758996db-3613-4805-b6ec-f34b16c3728e {"id":"758996db-3613-4805-b6ec-f34b16c3728e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d798ec1 758996db-3613-4805-b6ec-f34b16c3728e /tidb/cdc/default/default/upstream/7374684219425185212 {"id":7374684219425185212,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/758996db-3613-4805-b6ec-f34b16c3728e {"id":"758996db-3613-4805-b6ec-f34b16c3728e","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d798ec1 758996db-3613-4805-b6ec-f34b16c3728e /tidb/cdc/default/default/upstream/7374684219425185212 {"id":7374684219425185212,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.cli.2545.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-big-txn-test-10253?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2566.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: d55c8056-63df-41b0-a227-7c581dc7f03e Info: {"upstream_id":7374684219425185212,"namespace":"default","id":"d55c8056-63df-41b0-a227-7c581dc7f03e","sink_uri":"kafka://127.0.0.1:9092/ticdc-big-txn-test-10253?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:01:31.592974892+08:00","start_ts":450115008291930116,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115008291930116,"checkpoint_ts":450115008291930116,"checkpoint_time":"2024-05-30 15:01:31.348"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.12261.out cli tso query --pd=http://127.0.0.1:2379 < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:32 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ee95e14-8141-4d83-a6e2-9a668256f808 {"id":"6ee95e14-8141-4d83-a6e2-9a668256f808","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d7de3bd 6ee95e14-8141-4d83-a6e2-9a668256f808 /tidb/cdc/default/default/upstream/7374684225468360448 {"id":7374684225468360448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ee95e14-8141-4d83-a6e2-9a668256f808 {"id":"6ee95e14-8141-4d83-a6e2-9a668256f808","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d7de3bd 6ee95e14-8141-4d83-a6e2-9a668256f808 /tidb/cdc/default/default/upstream/7374684225468360448 {"id":7374684225468360448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ee95e14-8141-4d83-a6e2-9a668256f808 {"id":"6ee95e14-8141-4d83-a6e2-9a668256f808","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d7de3bd 6ee95e14-8141-4d83-a6e2-9a668256f808 /tidb/cdc/default/default/upstream/7374684225468360448 {"id":7374684225468360448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 6b8290aa-95fb-4b57-b479-1713eadf4e78 Info: {"upstream_id":7374684225468360448,"namespace":"default","id":"6b8290aa-95fb-4b57-b479-1713eadf4e78","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-30T15:01:32.714971031+08:00","start_ts":450115007770525697,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115007770525697,"checkpoint_ts":450115007770525697,"checkpoint_time":"2024-05-30 15:01:29.359"} [Thu May 30 15:01:32 CST 2024] <<<<<< START kafka consumer in charset_gbk case >>>>>> + set +x + tso='450115008273317889 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115008273317889 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:01:32 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.25922595.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check diff successfully ***************** properties ***************** "mysql.port"="4000" "insertproportion"="0" "recordcount"="20" "mysql.user"="root" "dotransactions"="false" "operationcount"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "updateproportion"="0" "readproportion"="0" "readallfields"="true" "threadcount"="2" "workload"="core" "mysql.db"="multi_capture_1" "requestdistribution"="uniform" ********************************************** Run finished, takes 14.891496ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1514.8, Avg(us): 1295, Min(us): 563, Max(us): 2456, 95th(us): 3000, 99th(us): 3000 ***************** properties ***************** "updateproportion"="0" "scanproportion"="0" "workload"="core" "insertproportion"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "recordcount"="20" "readproportion"="0" "mysql.port"="4000" "mysql.user"="root" "dotransactions"="false" "requestdistribution"="uniform" "mysql.db"="multi_capture_2" "threadcount"="2" "operationcount"="0" ********************************************** Run finished, takes 13.564918ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1636.9, Avg(us): 1148, Min(us): 525, Max(us): 2682, 95th(us): 2000, 99th(us): 3000 ***************** properties ***************** "recordcount"="20" "readproportion"="0" "workload"="core" "readallfields"="true" "dotransactions"="false" "updateproportion"="0" "operationcount"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "insertproportion"="0" "threadcount"="2" "mysql.port"="4000" "mysql.db"="multi_capture_3" "requestdistribution"="uniform" ********************************************** Run finished, takes 12.740019ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1731.1, Avg(us): 1227, Min(us): 500, Max(us): 1915, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.db"="multi_capture_4" "requestdistribution"="uniform" "threadcount"="2" "scanproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "readallfields"="true" "insertproportion"="0" "dotransactions"="false" "mysql.port"="4000" "recordcount"="20" "workload"="core" "readproportion"="0" "updateproportion"="0" ********************************************** Run finished, takes 15.155266ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1459.5, Avg(us): 1293, Min(us): 666, Max(us): 2073, 95th(us): 2000, 99th(us): 3000 check diff failed 1-th time, retry later ***************** properties ***************** "mysql.user"="root" "readallfields"="true" "workload"="core" "readproportion"="0" "fieldcount"="100" "insertproportion"="0" "recordcount"="5000" "requestdistribution"="uniform" "mysql.port"="4000" "scanproportion"="0" "updateproportion"="0" "mysql.db"="big_txn" "operationcount"="0" "threadcount"="1" "mysql.host"="127.0.0.1" "dotransactions"="false" ********************************************** ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:33 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17abd983-2824-4f1e-870e-2c3673f9a000 {"id":"17abd983-2824-4f1e-870e-2c3673f9a000","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d798bc5 17abd983-2824-4f1e-870e-2c3673f9a000 /tidb/cdc/default/default/upstream/7374684221109503614 {"id":7374684221109503614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17abd983-2824-4f1e-870e-2c3673f9a000 {"id":"17abd983-2824-4f1e-870e-2c3673f9a000","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d798bc5 17abd983-2824-4f1e-870e-2c3673f9a000 /tidb/cdc/default/default/upstream/7374684221109503614 {"id":7374684221109503614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17abd983-2824-4f1e-870e-2c3673f9a000 {"id":"17abd983-2824-4f1e-870e-2c3673f9a000","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d798bc5 17abd983-2824-4f1e-870e-2c3673f9a000 /tidb/cdc/default/default/upstream/7374684221109503614 {"id":7374684221109503614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2550.out cli changefeed create --start-ts=450115007596986369 '--sink-uri=kafka://127.0.0.1:9092/ticdc-tiflash-test-28645?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) Create changefeed successfully! ID: 98f07bb8-de4f-48d7-be07-3fda089e9360 Info: {"upstream_id":7374684221109503614,"namespace":"default","id":"98f07bb8-de4f-48d7-be07-3fda089e9360","sink_uri":"kafka://127.0.0.1:9092/ticdc-tiflash-test-28645?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:01:33.855321488+08:00","start_ts":450115007596986369,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115007596986369,"checkpoint_ts":450115007596986369,"checkpoint_time":"2024-05-30 15:01:28.697"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + tso='450115008603881473 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115008603881473 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:01:33 CST 2024] <<<<<< START cdc server in multi_topics_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.1228812290.out server --log-file /tmp/tidb_cdc_test/multi_topics_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_topics_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // cache Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } check diff failed 2-th time, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later [Pipeline] // withEnv [Pipeline] } check diff successfully [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + set +x [Thu May 30 15:01:35 CST 2024] <<<<<< START kafka consumer in tiflash case >>>>>> wait process cdc.test exit for 1-th time... table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:35 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c771335-f31d-4c22-a33d-246d7dafee34 {"id":"5c771335-f31d-4c22-a33d-246d7dafee34","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d8a12c2 5c771335-f31d-4c22-a33d-246d7dafee34 /tidb/cdc/default/default/upstream/7374684235175818312 {"id":7374684235175818312,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c771335-f31d-4c22-a33d-246d7dafee34 {"id":"5c771335-f31d-4c22-a33d-246d7dafee34","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d8a12c2 5c771335-f31d-4c22-a33d-246d7dafee34 /tidb/cdc/default/default/upstream/7374684235175818312 {"id":7374684235175818312,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c771335-f31d-4c22-a33d-246d7dafee34 {"id":"5c771335-f31d-4c22-a33d-246d7dafee34","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d8a12c2 5c771335-f31d-4c22-a33d-246d7dafee34 /tidb/cdc/default/default/upstream/7374684235175818312 {"id":7374684235175818312,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2640.out cli changefeed create --start-ts=450115008273317889 '--sink-uri=kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-7387?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: da1981bc-bb90-4045-8483-93e419de3e58 Info: {"upstream_id":7374684235175818312,"namespace":"default","id":"da1981bc-bb90-4045-8483-93e419de3e58","sink_uri":"kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-7387?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:01:36.389032613+08:00","start_ts":450115008273317889,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115008273317889,"checkpoint_ts":450115008273317889,"checkpoint_time":"2024-05-30 15:01:31.277"} PASS wait process cdc.test exit for 3-th time... coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later check diff failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:01:36 CST 2024] <<<<<< run test case multi_capture success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:37 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb728a06-8113-461f-aabc-40a77621bcba {"id":"eb728a06-8113-461f-aabc-40a77621bcba","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d8eccc7 eb728a06-8113-461f-aabc-40a77621bcba /tidb/cdc/default/default/upstream/7374684240191826646 {"id":7374684240191826646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb728a06-8113-461f-aabc-40a77621bcba {"id":"eb728a06-8113-461f-aabc-40a77621bcba","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d8eccc7 eb728a06-8113-461f-aabc-40a77621bcba /tidb/cdc/default/default/upstream/7374684240191826646 {"id":7374684240191826646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb728a06-8113-461f-aabc-40a77621bcba {"id":"eb728a06-8113-461f-aabc-40a77621bcba","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84d8eccc7 eb728a06-8113-461f-aabc-40a77621bcba /tidb/cdc/default/default/upstream/7374684240191826646 {"id":7374684240191826646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.12335.out cli changefeed create --start-ts=450115008603881473 '--sink-uri=kafka://127.0.0.1:9092/multi_topics?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/conf/changefeed.toml Create changefeed successfully! ID: a626c807-8247-43ea-95d0-b61a9a20de3a Info: {"upstream_id":7374684240191826646,"namespace":"default","id":"a626c807-8247-43ea-95d0-b61a9a20de3a","sink_uri":"kafka://127.0.0.1:9092/multi_topics?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-05-30T15:01:37.565112059+08:00","start_ts":450115008603881473,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["test.*"],"topic":"{schema}_{table}"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115008603881473,"checkpoint_ts":450115008603881473,"checkpoint_time":"2024-05-30 15:01:32.538"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Thu May 30 15:01:37 CST 2024] <<<<<< START kafka consumer in many_pk_or_uk case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240527072219-a97e6464c01d go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240527072219-a97e6464c01d go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 table cdc_tiflash_test.multi_data_type not exists for 2-th check, retry later go: downloading go.uber.org/multierr v1.11.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213772fc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:7120, start at 2024-05-30 15:01:37.136809034 +0800 CST m=+6.372574058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:37.147 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:37.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:37.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213772fc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:7120, start at 2024-05-30 15:01:37.136809034 +0800 CST m=+6.372574058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:37.147 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:37.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:37.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21376b200013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:7197, start at 2024-05-30 15:01:36.609564092 +0800 CST m=+5.692734168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:36.617 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:36.584 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:36.584 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.8060.out cli changefeed pause -c simple-handle-key-only Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... check diff failed 4-th time, retry later + set +x table cdc_tiflash_test.multi_data_type exists check diff successfully go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/text v0.15.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240522024016-df42997c2c57 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240510070134-e7f530c64790 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/google/uuid v1.6.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/tipb v0.0.0-20240507090649-2bf6bb0cb996 go: downloading github.com/tikv/pd/client v0.0.0-20240210135946-3488a653ddd9 go: downloading github.com/prometheus/client_golang v1.19.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/coocood/freecache v1.2.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/google/btree v1.1.2 wait process cdc.test exit for 1-th time... go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading golang.org/x/sys v0.20.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading golang.org/x/time v0.5.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading cloud.google.com/go/storage v1.39.1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.44.259 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.8080.out cli changefeed update -c simple-handle-key-only '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-11225?protocol=simple&max-message-bytes=700' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/conf/changefeed.toml --no-confirm + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.8403.out cli tso query --pd=http://127.0.0.1:2379 table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.6 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading cloud.google.com/go v0.112.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/net v0.25.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/pkg/errors v0.9.1 go: downloading golang.org/x/tools v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading google.golang.org/protobuf v1.34.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/golang/snappy v0.0.4 check diff failed 2-th time, retry later Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/simple-handle-key-only-11225?protocol=simple To:kafka://127.0.0.1:9092/simple-handle-key-only-11225?protocol=simple&max-message-bytes=700} {Type:update Path:[Config SyncPointInterval] From: To:0xc003595e08} {Type:update Path:[Config SyncPointRetention] From: To:0xc003595e18} {Type:update Path:[Config Consistent] From: To:0xc00144e230} check diff failed 5-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:01:41 CST 2024] <<<<<< run test case tiflash success! >>>>>> Update changefeed config successfully! ID: simple-handle-key-only Info: {"upstream_id":7374684198568163077,"namespace":"default","id":"simple-handle-key-only","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-11225?protocol=simple\u0026max-message-bytes=700","create_time":"2024-05-30T15:01:30.552619862+08:00","start_ts":450115007534333954,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115010064285701,"checkpoint_ts":450115010064285701,"checkpoint_time":"2024-05-30 15:01:38.109"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/procfs v0.15.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/sasha-s/go-deadlock v0.3.1 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2137a44c0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:11001, start at 2024-05-30 15:01:40.264647798 +0800 CST m=+6.367269433 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:40.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:40.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:40.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2137a44c0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:11001, start at 2024-05-30 15:01:40.264647798 +0800 CST m=+6.367269433 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:40.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:40.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:40.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2137a8340014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:11058, start at 2024-05-30 15:01:40.535902616 +0800 CST m=+6.533843221 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:03:40.545 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:01:40.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:51:40.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 + set +x + tso='450115010844950530 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115010844950530 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:01:42 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.84358437.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading github.com/rivo/uniseg v0.4.7 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.8106.out cli changefeed resume -c simple-handle-key-only go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 check diff failed 3-th time, retry later check diff successfully go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/ncw/directio v1.0.5 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 table test.finish_mark not exists for 2-th check, retry later + set +x check diff failed 4-th time, retry later check diff failed 1-th time, retry later [Thu May 30 15:01:44 CST 2024] <<<<<< START cdc server in mq_sink_dispatcher case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.1227212274.out server --log-file /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc.log --log-level info --data-dir /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 1-th check, retry later INSERT - Takes(s): 10.0, Count: 2402, OPS: 239.6, Avg(us): 3868, Min(us): 1304, Max(us): 95330, 95th(us): 3000, 99th(us): 84000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:45 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/106a054e-e1e4-4aab-9e9c-463b862c7cf1 {"id":"106a054e-e1e4-4aab-9e9c-463b862c7cf1","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84dac18c5 106a054e-e1e4-4aab-9e9c-463b862c7cf1 /tidb/cdc/default/default/upstream/7374684268793752398 {"id":7374684268793752398,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/106a054e-e1e4-4aab-9e9c-463b862c7cf1 {"id":"106a054e-e1e4-4aab-9e9c-463b862c7cf1","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84dac18c5 106a054e-e1e4-4aab-9e9c-463b862c7cf1 /tidb/cdc/default/default/upstream/7374684268793752398 {"id":7374684268793752398,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/106a054e-e1e4-4aab-9e9c-463b862c7cf1 {"id":"106a054e-e1e4-4aab-9e9c-463b862c7cf1","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84dac18c5 106a054e-e1e4-4aab-9e9c-463b862c7cf1 /tidb/cdc/default/default/upstream/7374684268793752398 {"id":7374684268793752398,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.8484.out cli changefeed create --start-ts=450115010844950530 '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-18953?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: aa0066bc-4848-442c-9b6e-fe2047d8f2c5 Info: {"upstream_id":7374684268793752398,"namespace":"default","id":"aa0066bc-4848-442c-9b6e-fe2047d8f2c5","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-18953?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:01:46.109347501+08:00","start_ts":450115010844950530,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115010844950530,"checkpoint_ts":450115010844950530,"checkpoint_time":"2024-05-30 15:01:41.087"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 3-th check, retry later check diff failed 5-th time, retry later + set +x [Thu May 30 15:01:47 CST 2024] <<<<<< START kafka consumer in ddl_sequence case >>>>>> check diff failed 2-th time, retry later go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:01:47 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f11d70a3-b7ca-4290-8579-04ad98d30096 {"id":"f11d70a3-b7ca-4290-8579-04ad98d30096","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84db9c8c8 f11d70a3-b7ca-4290-8579-04ad98d30096 /tidb/cdc/default/default/upstream/7374684282925678793 {"id":7374684282925678793,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f11d70a3-b7ca-4290-8579-04ad98d30096 {"id":"f11d70a3-b7ca-4290-8579-04ad98d30096","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84db9c8c8 f11d70a3-b7ca-4290-8579-04ad98d30096 /tidb/cdc/default/default/upstream/7374684282925678793 {"id":7374684282925678793,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f11d70a3-b7ca-4290-8579-04ad98d30096 {"id":"f11d70a3-b7ca-4290-8579-04ad98d30096","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84db9c8c8 f11d70a3-b7ca-4290-8579-04ad98d30096 /tidb/cdc/default/default/upstream/7374684282925678793 {"id":7374684282925678793,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 4-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.12345.out cli tso query --pd=http://127.0.0.1:2379 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) check diff successfully check diff failed 1-th time, retry later table ddl_sequence.finish_mark not exists for 1-th check, retry later [Pipeline] // cache check diff failed 3-th time, retry later [Pipeline] } [Pipeline] // dir [Pipeline] } table test.finish_mark exists check diff successfully [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table test.finish_mark not exists for 5-th check, retry later [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv wait process cdc.test exit for 2-th time... [Pipeline] } [Pipeline] // stage [Pipeline] } + set +x + tso='450115012956258306 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115012956258306 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.12378.out cli changefeed create --start-ts=450115012956258306 '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/changefeed.toml wait process cdc.test exit for 3-th time... Create changefeed successfully! ID: test Info: {"upstream_id":7374684282925678793,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-30T15:01:51.109382965+08:00","start_ts":450115012956258306,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["verify.t"],"partition":"index-value"},{"matcher":["dispatcher.index"],"partition":"index-value","index":"idx_a"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115012956258306,"checkpoint_ts":450115012956258306,"checkpoint_time":"2024-05-30 15:01:49.141"} PASS check diff failed 4-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:01:51 CST 2024] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> check diff successfully table ddl_sequence.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115012956258306, "checkpoint_time": "2024-05-30 15:01:49.141", "error": null }' + echo '{ "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115012956258306, "checkpoint_time": "2024-05-30 15:01:49.141", "error": null }' { "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115012956258306, "checkpoint_time": "2024-05-30 15:01:49.141", "error": null } ++ echo '{' '"upstream_id":' 7374684282925678793, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450115012956258306, '"checkpoint_time":' '"2024-05-30' '15:01:49.141",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374684282925678793, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450115012956258306, '"checkpoint_time":' '"2024-05-30' '15:01:49.141",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... Run finished, takes 19.340748302s INSERT - Takes(s): 19.3, Count: 5000, OPS: 258.6, Avg(us): 3583, Min(us): 1289, Max(us): 95330, 95th(us): 4000, 99th(us): 82000 [Thu May 30 15:01:52 CST 2024] <<<<<< START kafka consumer in big_txn case >>>>>> table big_txn.usertable not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:01:53 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> wait process cdc.test exit for 3-th time... check diff failed 5-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:01:54 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> table ddl_sequence.finish_mark not exists for 3-th check, retry later check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115012956258306, "checkpoint_time": "2024-05-30 15:01:49.141", "error": null }' + echo '{ "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115012956258306, "checkpoint_time": "2024-05-30 15:01:49.141", "error": null }' { "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115012956258306, "checkpoint_time": "2024-05-30 15:01:49.141", "error": null } ++ echo '{' '"upstream_id":' 7374684282925678793, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450115012956258306, '"checkpoint_time":' '"2024-05-30' '15:01:49.141",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 1-th time, retry later table big_txn.usertable not exists for 2-th check, retry later check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_sequence.finish_mark not exists for 4-th check, retry later table big_txn.usertable exists check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115013978619909, "checkpoint_time": "2024-05-30 15:01:53.041", "error": null }' + echo '{ "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115013978619909, "checkpoint_time": "2024-05-30 15:01:53.041", "error": null }' { "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115013978619909, "checkpoint_time": "2024-05-30 15:01:53.041", "error": null } ++ echo '{' '"upstream_id":' 7374684282925678793, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450115013978619909, '"checkpoint_time":' '"2024-05-30' '15:01:53.041",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 2-th time, retry later check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table ddl_sequence.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:01:59 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> check diff failed 2-th time, retry later Verifying downstream PD is started... check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 450115014136168464, "checkpoint_time": "2024-05-30 15:01:53.642", "error": { "time": "2024-05-30T15:01:58.372308538+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a" } }' + echo '{ "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 450115014136168464, "checkpoint_time": "2024-05-30 15:01:53.642", "error": { "time": "2024-05-30T15:01:58.372308538+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a" } }' { "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 450115014136168464, "checkpoint_time": "2024-05-30 15:01:53.642", "error": { "time": "2024-05-30T15:01:58.372308538+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7374684282925678793, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 450115014136168464, '"checkpoint_time":' '"2024-05-30' '15:01:53.642",' '"error":' '{' '"time":' '"2024-05-30T15:01:58.372308538+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: dispatcher.index, index: 'idx_a"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7374684282925678793, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 450115014136168464, '"checkpoint_time":' '"2024-05-30' '15:01:53.642",' '"error":' '{' '"time":' '"2024-05-30T15:01:58.372308538+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: dispatcher.index, index: 'idx_a"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a' + [[ ! [CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a =~ ErrDispatcherFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.12581.out cli changefeed update -c test '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/new_changefeed.toml --no-confirm Diff of changefeed config: {Type:update Path:[Config SyncPointInterval] From: To:0xc00351e7a8} {Type:update Path:[Config SyncPointRetention] From: To:0xc00351e7b8} {Type:update Path:[Config Sink DispatchRules 0 Matcher 0] From:verify.t To:dispatcher.index} {Type:delete Path:[Config Sink DispatchRules 1 Matcher 0] From:dispatcher.index To:} {Type:delete Path:[Config Sink DispatchRules 1 PartitionRule] From:index-value To:} {Type:delete Path:[Config Sink DispatchRules 1 IndexName] From:idx_a To:} {Type:update Path:[Config Consistent] From: To:0xc000e3df10} Update changefeed config successfully! ID: test Info: {"upstream_id":7374684282925678793,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-30T15:01:51.109382965+08:00","start_ts":450115012956258306,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","dispatchers":[{"matcher":["dispatcher.index"],"partition":"index-value"}],"encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"failed","error":{"addr":"127.0.0.1:8300","code":"CDC:ErrDispatcherFailed","message":"[CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a"},"creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115015276494859,"checkpoint_ts":450115014136168464,"checkpoint_time":"2024-05-30 15:01:53.642"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 3-th time, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.12609.out cli changefeed resume -c test PASS /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh: line 1: 8129 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 4-th time, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115017111502856, "checkpoint_time": "2024-05-30 15:02:04.992", "error": null }' + echo '{ "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115017111502856, "checkpoint_time": "2024-05-30 15:02:04.992", "error": null }' { "upstream_id": 7374684282925678793, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450115017111502856, "checkpoint_time": "2024-05-30 15:02:04.992", "error": null } ++ echo '{' '"upstream_id":' 7374684282925678793, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450115017111502856, '"checkpoint_time":' '"2024-05-30' '15:02:04.992",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374684282925678793, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450115017111502856, '"checkpoint_time":' '"2024-05-30' '15:02:04.992",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... check diff failed 5-th time, retry later Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... check diff successfully table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Thu May 30 15:02:09 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213977d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn, pid:3613, start at 2024-05-30 15:02:10.199125225 +0800 CST m=+6.042719406 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:10.206 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:10.165 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:10.165 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213977d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn, pid:3613, start at 2024-05-30 15:02:10.199125225 +0800 CST m=+6.042719406 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:10.206 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:10.165 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:10.165 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... check diff failed 2-th time, retry later Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213985040014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn, pid:3674, start at 2024-05-30 15:02:11.043355095 +0800 CST m=+6.793780587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:11.050 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:11.009 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:11.009 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Thu May 30 15:02:14 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.49654967.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:02:18 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5f5370a-5f83-4b70-af6a-683ba4284cd5 {"id":"a5f5370a-5f83-4b70-af6a-683ba4284cd5","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e303790 a5f5370a-5f83-4b70-af6a-683ba4284cd5 /tidb/cdc/default/default/upstream/7374684418580034802 {"id":7374684418580034802,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5f5370a-5f83-4b70-af6a-683ba4284cd5 {"id":"a5f5370a-5f83-4b70-af6a-683ba4284cd5","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e303790 a5f5370a-5f83-4b70-af6a-683ba4284cd5 /tidb/cdc/default/default/upstream/7374684418580034802 {"id":7374684418580034802,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5f5370a-5f83-4b70-af6a-683ba4284cd5 {"id":"a5f5370a-5f83-4b70-af6a-683ba4284cd5","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e303790 a5f5370a-5f83-4b70-af6a-683ba4284cd5 /tidb/cdc/default/default/upstream/7374684418580034802 {"id":7374684418580034802,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.cli.5018.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-region-merge-test-15552?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 0ff0cfd5-11e4-429a-a769-2e5e6e0a507c Info: {"upstream_id":7374684418580034802,"namespace":"default","id":"0ff0cfd5-11e4-429a-a769-2e5e6e0a507c","sink_uri":"kafka://127.0.0.1:9092/ticdc-region-merge-test-15552?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:02:18.531560771+08:00","start_ts":450115020617416707,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115020617416707,"checkpoint_ts":450115020617416707,"checkpoint_time":"2024-05-30 15:02:18.366"} PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2139f2b80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3, pid:3920, start at 2024-05-30 15:02:18.076459016 +0800 CST m=+5.864462419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:18.084 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:18.080 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:18.080 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 5-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x [Thu May 30 15:02:19 CST 2024] <<<<<< START kafka consumer in region_merge case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2139f3640010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:9096, start at 2024-05-30 15:02:18.095173493 +0800 CST m=+6.400847603 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:18.103 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:18.073 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:18.073 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2139f3640010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:9096, start at 2024-05-30 15:02:18.095173493 +0800 CST m=+6.400847603 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:18.103 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:18.073 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:18.073 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2139f228000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:9157, start at 2024-05-30 15:02:18.014858058 +0800 CST m=+6.218797064 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:18.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:17.994 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:17.994 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2139f2b80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3, pid:3920, start at 2024-05-30 15:02:18.076459016 +0800 CST m=+5.864462419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:18.084 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:18.080 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:18.080 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2139f75c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3, pid:3997, start at 2024-05-30 15:02:18.367087497 +0800 CST m=+6.006179844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:18.378 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:18.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:18.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 6-th time, retry later split_and_random_merge scale: 20 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh: line 1: 12669 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/new_changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [Thu May 30 15:02:22 CST 2024] <<<<<< START cdc server in kafka_simple_handle_key_only_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.1042910431.out server --log-file /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully [Thu May 30 15:02:23 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.52145216.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector Starting Upstream PD... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:02:25 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f259e56-e5f5-4484-83a5-ad386ba55027 {"id":"7f259e56-e5f5-4484-83a5-ad386ba55027","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e4fc7c3 7f259e56-e5f5-4484-83a5-ad386ba55027 /tidb/cdc/default/default/upstream/7374684451016225948 {"id":7374684451016225948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f259e56-e5f5-4484-83a5-ad386ba55027 {"id":"7f259e56-e5f5-4484-83a5-ad386ba55027","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e4fc7c3 7f259e56-e5f5-4484-83a5-ad386ba55027 /tidb/cdc/default/default/upstream/7374684451016225948 {"id":7374684451016225948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f259e56-e5f5-4484-83a5-ad386ba55027 {"id":"7f259e56-e5f5-4484-83a5-ad386ba55027","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e4fc7c3 7f259e56-e5f5-4484-83a5-ad386ba55027 /tidb/cdc/default/default/upstream/7374684451016225948 {"id":7374684451016225948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.10478.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:02:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b019a07-c6e9-42c6-9c45-fbbd8141ed64 {"id":"5b019a07-c6e9-42c6-9c45-fbbd8141ed64","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e55cd0c 5b019a07-c6e9-42c6-9c45-fbbd8141ed64 /tidb/cdc/default/default/upstream/7374684454509592560 {"id":7374684454509592560,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b019a07-c6e9-42c6-9c45-fbbd8141ed64 {"id":"5b019a07-c6e9-42c6-9c45-fbbd8141ed64","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e55cd0c 5b019a07-c6e9-42c6-9c45-fbbd8141ed64 /tidb/cdc/default/default/upstream/7374684454509592560 {"id":7374684454509592560,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b019a07-c6e9-42c6-9c45-fbbd8141ed64 {"id":"5b019a07-c6e9-42c6-9c45-fbbd8141ed64","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e55cd0c 5b019a07-c6e9-42c6-9c45-fbbd8141ed64 /tidb/cdc/default/default/upstream/7374684454509592560 {"id":7374684454509592560,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5268.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-batch-add-table-test-1700?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + set +x + tso='450115022506426373 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115022506426373 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.10515.out cli changefeed create --start-ts=450115022506426373 '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-avro-19071?protocol=simple&encoding-format=avro' -c simple-handle-key-only-avro --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/conf/changefeed.toml ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later Create changefeed successfully! ID: b9490b8c-2d1e-4f9c-af42-58a627d75378 Info: {"upstream_id":7374684454509592560,"namespace":"default","id":"b9490b8c-2d1e-4f9c-af42-58a627d75378","sink_uri":"kafka://127.0.0.1:9092/ticdc-batch-add-table-test-1700?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:02:27.365662312+08:00","start_ts":450115022927953923,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115022927953923,"checkpoint_ts":450115022927953923,"checkpoint_time":"2024-05-30 15:02:27.180"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... *************************** 1. row *************************** count(distinct region_id): 1 Create changefeed successfully! ID: simple-handle-key-only-avro Info: {"upstream_id":7374684451016225948,"namespace":"default","id":"simple-handle-key-only-avro","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-avro-19071?protocol=simple\u0026encoding-format=avro","create_time":"2024-05-30T15:02:27.581912127+08:00","start_ts":450115022506426373,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115022506426373,"checkpoint_ts":450115022506426373,"checkpoint_time":"2024-05-30 15:02:25.572"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213a85b00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr, pid:3608, start at 2024-05-30 15:02:27.459933904 +0800 CST m=+5.862218377 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:27.468 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:27.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:27.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213a85b00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr, pid:3608, start at 2024-05-30 15:02:27.459933904 +0800 CST m=+5.862218377 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:27.468 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:27.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:27.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213a82840014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr, pid:3688, start at 2024-05-30 15:02:27.269107813 +0800 CST m=+5.557848058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:27.275 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:27.233 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:27.233 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check diff failed 3-th time, retry later + set +x [Thu May 30 15:02:28 CST 2024] <<<<<< START kafka consumer in batch_add_table case >>>>>> table batch_add_table.finish_mark not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213aa7380012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:9545, start at 2024-05-30 15:02:29.605871254 +0800 CST m=+7.276282656 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:29.612 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:29.582 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:29.582 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later table batch_add_table.finish_mark not exists for 2-th check, retry later split_and_random_merge scale: 40 [Thu May 30 15:02:31 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.49754977.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213aa7380012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:9545, start at 2024-05-30 15:02:29.605871254 +0800 CST m=+7.276282656 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:29.612 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:29.582 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:29.582 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213aa8d8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-2mtqs, pid:9616, start at 2024-05-30 15:02:29.699232511 +0800 CST m=+5.297063288 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:29.707 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:29.686 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:29.686 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table batch_add_table.finish_mark exists check diff successfully [2024/05/30 15:02:27.719 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/05/30 15:02:27.719 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/05/30 15:02:28.266 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8ac76e54_6d0e_432a_b9cb_20a7089652ec"] [2024/05/30 15:02:28.273 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs007123a4_738c_4bbf_a22a_2bec8128ec63"] [2024/05/30 15:02:28.275 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc609acff_b47c_48fe_8ee1_59f557570028"] [2024/05/30 15:02:28.277 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs07390ddd_8674_4e96_a498_45b2bea3b1e5"] [2024/05/30 15:02:28.278 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc98b52cd_e2a3_4505_a0c0_e4db6ee2f6e2"] [2024/05/30 15:02:28.281 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4dc04b6f_e6cd_4980_80a4_233a5767247c"] [2024/05/30 15:02:28.283 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsae94968a_3ead_47b4_a8d7_30a3ae431b06"] [2024/05/30 15:02:28.797 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs72b29c2f_b9bb_484c_847c_1e023f1a843f"] [2024/05/30 15:02:29.420 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/30 15:02:29.707 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:29.708 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:29.711 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:29.711 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:29.712 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:29.712 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:29.712 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:29.797 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/30 15:02:29.820 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:29.820 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:29.896 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:29.896 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:29.896 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:29.899 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:29.902 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:29.996 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/30 15:02:30.016 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:30.209 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:30.211 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/30 15:02:30.803 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/30 15:02:31.199 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:02:31.204 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:02:31.206 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:02:31.210 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:02:31.211 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:02:31.213 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:02:31.217 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/30 15:02:31.293 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:02:31.395 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:02:31.404 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:02:31.413 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:02:31.413 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:02:31.418 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:02:31.419 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:02:31.495 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:02:31.521 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:02:31.712 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:02:32.106 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/30 15:02:32.112 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/30 15:02:32.216 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/30 15:02:32.299 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/30 15:02:32.400 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/30 15:02:32.706 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:02:32.794 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:02:32.803 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:02:32.803 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:02:32.806 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:02:32.806 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:02:32.810 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/30 15:02:32.814 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:02:33.012 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:02:33.021 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:02:33.112 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:02:33.193 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:02:33.193 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:02:33.201 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:02:33.204 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:02:33.210 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:02:33.317 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] check diff failed 5-th time, retry later [2024/05/30 15:02:33.714 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] wait process cdc.test exit for 1-th time... [2024/05/30 15:02:34.005 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:02:34.007 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:02:34.096 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:02:34.098 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:02:34.100 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/30 15:02:34.101 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:02:34.102 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:02:34.108 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] wait process cdc.test exit for 2-th time... [2024/05/30 15:02:34.204 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/30 15:02:34.211 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:02:34.218 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/30 15:02:34.399 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:02:34.605 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:02:34.609 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] wait process cdc.test exit for 3-th time... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.10836.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/30 15:02:34.697 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:02:34.701 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:02:34.704 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:02:34.709 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:02:34.801 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.10569.out cli changefeed pause -c simple-handle-key-only-avro + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:02:34 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77b880dc-cf95-4e99-b904-56505aa2a129 {"id":"77b880dc-cf95-4e99-b904-56505aa2a129","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7c7dbe 77b880dc-cf95-4e99-b904-56505aa2a129 /tidb/cdc/default/default/upstream/7374684498643733289 {"id":7374684498643733289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77b880dc-cf95-4e99-b904-56505aa2a129 {"id":"77b880dc-cf95-4e99-b904-56505aa2a129","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7c7dbe 77b880dc-cf95-4e99-b904-56505aa2a129 /tidb/cdc/default/default/upstream/7374684498643733289 {"id":7374684498643733289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77b880dc-cf95-4e99-b904-56505aa2a129 {"id":"77b880dc-cf95-4e99-b904-56505aa2a129","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7c7dbe 77b880dc-cf95-4e99-b904-56505aa2a129 /tidb/cdc/default/default/upstream/7374684498643733289 {"id":7374684498643733289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5024.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c=ddl-manager Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7374684498643733289,"namespace":"default","id":"ddl-manager","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:02:34.965718334+08:00","start_ts":450115024922083335,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115024922083335,"checkpoint_ts":450115024922083335,"checkpoint_time":"2024-05-30 15:02:34.787"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:02:35 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> [2024/05/30 15:02:35.210 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] coverage: 1.9% of statements in github.com/pingcap/tiflow/... check diff successfully [2024/05/30 15:02:35.512 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:02:35.602 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:02:35.617 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:02:35.696 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:02:35.701 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/30 15:02:35.704 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:02:35.708 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:02:35.709 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:02:35.803 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:02:36.011 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + set +x [Thu May 30 15:02:36 CST 2024] <<<<<< START kafka consumer in ddl_manager case >>>>>> [2024/05/30 15:02:36.300 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:02:36.306 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:02:36.312 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:02:36.318 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:02:36.318 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:02:36.406 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + set +x + tso='450115024999415809 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115024999415809 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:02:36 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1086510869.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/30 15:02:36.504 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/30 15:02:36.516 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/30 15:02:36.519 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/30 15:02:36.596 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:02:36.598 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/30 15:02:36.714 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.10598.out cli changefeed update -c simple-handle-key-only-avro '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-avro-19071?protocol=simple&encoding-format=avro&max-message-bytes=650' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/conf/changefeed.toml --no-confirm [2024/05/30 15:02:36.917 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213af9a40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:13628, start at 2024-05-30 15:02:34.888391286 +0800 CST m=+6.393281951 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:34.971 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:34.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:34.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213af9a40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:13628, start at 2024-05-30 15:02:34.888391286 +0800 CST m=+6.393281951 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:34.971 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:34.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:34.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213aff400014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:13698, start at 2024-05-30 15:02:35.265728363 +0800 CST m=+6.656816273 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:04:35.274 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:02:35.266 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:52:35.266 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/30 15:02:37.015 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:02:37.116 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:02:37.201 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/30 15:02:37.206 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:02:37.209 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:02:37.214 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/simple-handle-key-only-avro-19071?protocol=simple&encoding-format=avro To:kafka://127.0.0.1:9092/simple-handle-key-only-avro-19071?protocol=simple&encoding-format=avro&max-message-bytes=650} {Type:update Path:[Config SyncPointInterval] From: To:0xc0034772f8} {Type:update Path:[Config SyncPointRetention] From: To:0xc003477308} {Type:update Path:[Config Consistent] From: To:0xc0010c0ee0} Update changefeed config successfully! ID: simple-handle-key-only-avro Info: {"upstream_id":7374684451016225948,"namespace":"default","id":"simple-handle-key-only-avro","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-avro-19071?protocol=simple\u0026encoding-format=avro\u0026max-message-bytes=650","create_time":"2024-05-30T15:02:27.581912127+08:00","start_ts":450115022506426373,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115024918413319,"checkpoint_ts":450115024918413319,"checkpoint_time":"2024-05-30 15:02:34.773"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:02:37.296 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:02:37.302 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] check diff failed 1-th time, retry later [2024/05/30 15:02:37.706 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:02:38.015 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:02:38.099 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:02:38.106 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:02:38.109 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:02:38.113 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:02:38.118 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:02:38.205 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] *************************** 1. row *************************** count(distinct region_id): 1 [2024/05/30 15:02:38.399 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/30 15:02:38.510 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.10622.out cli changefeed resume -c simple-handle-key-only-avro [2024/05/30 15:02:38.606 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:02:38.712 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/30 15:02:38.717 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:02:38.800 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/30 15:02:38.804 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:02:38.808 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:02:38.812 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:02:38.814 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/30 15:02:38.815 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:02:38.901 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:02:39.217 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] check diff failed 2-th time, retry later [Thu May 30 15:02:39 CST 2024] <<<<<< START cdc server in kafka_column_selector case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.1499714999.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:02:39.692 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:02:39.697 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:02:39.701 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:02:39.708 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:02:39.801 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:02:39.806 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:02:39.806 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:02:39 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4faf1114-06c2-4a06-85d2-264e3f4e89be {"id":"4faf1114-06c2-4a06-85d2-264e3f4e89be","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7763ca 4faf1114-06c2-4a06-85d2-264e3f4e89be /tidb/cdc/default/default/upstream/7374684494185676535 {"id":7374684494185676535,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4faf1114-06c2-4a06-85d2-264e3f4e89be {"id":"4faf1114-06c2-4a06-85d2-264e3f4e89be","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7763ca 4faf1114-06c2-4a06-85d2-264e3f4e89be /tidb/cdc/default/default/upstream/7374684494185676535 {"id":7374684494185676535,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4faf1114-06c2-4a06-85d2-264e3f4e89be {"id":"4faf1114-06c2-4a06-85d2-264e3f4e89be","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7763ca 4faf1114-06c2-4a06-85d2-264e3f4e89be /tidb/cdc/default/default/upstream/7374684494185676535 {"id":7374684494185676535,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.10917.out cli changefeed create --start-ts=450115024999415809 '--sink-uri=kafka://127.0.0.1:9092/ticdc-resourcecontrol-test-4453?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' [2024/05/30 15:02:40.103 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/30 15:02:40.125 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Create changefeed successfully! ID: 6610d551-81ad-45d4-9fea-c8cab742efb2 Info: {"upstream_id":7374684494185676535,"namespace":"default","id":"6610d551-81ad-45d4-9fea-c8cab742efb2","sink_uri":"kafka://127.0.0.1:9092/ticdc-resourcecontrol-test-4453?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:02:40.082502438+08:00","start_ts":450115024999415809,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115024999415809,"checkpoint_ts":450115024999415809,"checkpoint_time":"2024-05-30 15:02:35.082"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:02:40.213 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:02:40.316 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] wait process 4980 exit for 1-th time... wait process 4980 exit for 2-th time... [2024/05/30 15:02:40.406 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/30 15:02:40.415 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:02:40.505 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:02:40.511 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:02:40.516 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:02:40.519 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] wait process 4980 exit for 3-th time... + set +x table test.finish_mark not exists for 1-th check, retry later [2024/05/30 15:02:41.000 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/30 15:02:41.014 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/30 15:02:41.206 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] wait process 4980 exit for 4-th time... [2024/05/30 15:02:41.312 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/30 15:02:41.319 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] + set +x [Thu May 30 15:02:41 CST 2024] <<<<<< START kafka consumer in resourcecontrol case >>>>>> [2024/05/30 15:02:41.608 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:02:41.608 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:02:41.701 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:02:41.707 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/30 15:02:41.713 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] check diff failed 3-th time, retry later [2024/05/30 15:02:41.800 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:02:41.806 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:02:41.810 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:02:41.815 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:02:41.815 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:02:41.911 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] wait process 4980 exit for 5-th time... [2024/05/30 15:02:42.008 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/30 15:02:42.093 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:02:42.203 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:02:42.204 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:02:42.208 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:02:42.214 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:02:42 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/42db1e22-2bdb-4c79-892b-f3d82ed1db4d {"id":"42db1e22-2bdb-4c79-892b-f3d82ed1db4d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e9561c6 42db1e22-2bdb-4c79-892b-f3d82ed1db4d /tidb/cdc/default/default/upstream/7374684530008921959 {"id":7374684530008921959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' table resourcecontrol.finish_mark not exists for 1-th check, retry later + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/42db1e22-2bdb-4c79-892b-f3d82ed1db4d {"id":"42db1e22-2bdb-4c79-892b-f3d82ed1db4d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e9561c6 42db1e22-2bdb-4c79-892b-f3d82ed1db4d /tidb/cdc/default/default/upstream/7374684530008921959 {"id":7374684530008921959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/42db1e22-2bdb-4c79-892b-f3d82ed1db4d {"id":"42db1e22-2bdb-4c79-892b-f3d82ed1db4d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e9561c6 42db1e22-2bdb-4c79-892b-f3d82ed1db4d /tidb/cdc/default/default/upstream/7374684530008921959 {"id":7374684530008921959,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.cli.15043.out cli changefeed create --start-ts=450115026054807553 '--sink-uri=kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json&partition-num=1&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/conf/changefeed.toml wait process 4980 exit for 6-th time... Create changefeed successfully! ID: test Info: {"upstream_id":7374684530008921959,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json\u0026partition-num=1\u0026enable-tidb-extension=true","create_time":"2024-05-30T15:02:42.721369706+08:00","start_ts":450115026054807553,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":[{"matcher":["test.t1"],"columns":["a","b"]},{"matcher":["test.*"],"columns":["*","!b"]},{"matcher":["test1.t1"],"columns":["column*","!column1"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115026054807553,"checkpoint_ts":450115026054807553,"checkpoint_time":"2024-05-30 15:02:39.108"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process 4980 exit for 7-th time... [2024/05/30 15:02:43.006 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (4980) - No such process wait process 4980 exit for 8-th time... process 4980 already exit [Thu May 30 15:02:43 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> [2024/05/30 15:02:43.305 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/30 15:02:43.395 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:02:43.396 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:02:43.402 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] split_and_random_merge scale: 80 table test.finish_mark not exists for 2-th check, retry later + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.51375139.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/30 15:02:43.418 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/30 15:02:43.496 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:02:43.497 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:02:43.510 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:02:43.593 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:02:43.593 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:02:43.601 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:02:43.610 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] check diff failed 4-th time, retry later [2024/05/30 15:02:43.703 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/30 15:02:43.792 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:02:43.896 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/30 15:02:44.001 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:02:44.001 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:02:44.002 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:02:44.007 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] + set +x table resourcecontrol.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... [2024/05/30 15:02:45.008 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table test.finish_mark exists check diff successfully wait process cdc.test exit for 2-th time... [2024/05/30 15:02:45.397 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:02:45.398 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/30 15:02:45.409 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:02:45.494 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:02:45.508 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:02:45.516 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:02:45.516 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:02:45.606 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/30 15:02:45.610 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] check diff failed 5-th time, retry later wait process cdc.test exit for 1-th time... [2024/05/30 15:02:45.693 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:02:45.696 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:02:45.701 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:02:45.711 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:02:45.813 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:02:45.893 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:02:45.903 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:02:45.905 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:02:46.111 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:02:46 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> wait process cdc.test exit for 2-th time... [2024/05/30 15:02:46.194 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > wait process cdc.test exit for 3-th time... [2024/05/30 15:02:46.710 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/30 15:02:46.799 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:02:47 CST 2024] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> [2024/05/30 15:02:47.096 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:02:47.402 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:02:47.403 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/30 15:02:47.411 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:02:47.499 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:02:47.503 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:02:47.509 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:02:47.514 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/30 15:02:47.598 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:02:47.601 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:02:47.602 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:02:47.603 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:02:47.605 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/30 15:02:47.693 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:02:47.706 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:02:47.711 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:02:47.798 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:02:47.807 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] check diff failed 6-th time, retry later Starting build checksum checker... go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading golang.org/x/net v0.25.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240527072219-a97e6464c01d go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240527072219-a97e6464c01d go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/tikv/pd/client v0.0.0-20240210135946-3488a653ddd9 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240510070134-e7f530c64790 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240522024016-df42997c2c57 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/prometheus/client_golang v1.19.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading golang.org/x/time v0.5.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading golang.org/x/text v0.15.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/stretchr/objx v0.5.2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/spf13/pflag v1.0.5 go: downloading golang.org/x/sys v0.20.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 [2024/05/30 15:02:49.005 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/30 15:02:49.301 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.34.1 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/beorn7/perks v1.0.1 [2024/05/30 15:02:49.523 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/30 15:02:49.601 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/30 15:02:49.603 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/30 15:02:49.606 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:02:49.607 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:02:49.607 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/30 15:02:49.617 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:02:49.617 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.15.0 go: downloading github.com/aws/smithy-go v1.13.5 [2024/05/30 15:02:49.699 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:02:49.699 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/30 15:02:49.709 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:02:49.710 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/30 15:02:49.720 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/30 15:02:49.805 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... check diff successfully *************************** 1. row *************************** count(distinct region_id): 1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 wait process cdc.test exit for 1-th time... go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda [2024/05/30 15:02:50.499 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:02:50.619 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/30 15:02:50.703 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] wait process cdc.test exit for 2-th time... < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:02:50 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7374684498643733289 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:02:34.965718334 +0800 CST StartTs:450115024922083335 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004105200 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115024961404932} {CheckpointTs:450115025354620929 MinTableBarrierTs:450115025354620929 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/364783e3-e914-467f-98e5-8c575a7b7d95 {"id":"364783e3-e914-467f-98e5-8c575a7b7d95","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7c7ff1 364783e3-e914-467f-98e5-8c575a7b7d95 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374684498643733289,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:02:34.965718334+08:00","start-ts":450115024922083335,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115024961404932} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450115025354620929,"min-table-barrier-ts":450115025354620929,"admin-job-type":0} /tidb/cdc/default/default/task/position/364783e3-e914-467f-98e5-8c575a7b7d95/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684498643733289 {"id":7374684498643733289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7374684498643733289 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:02:34.965718334 +0800 CST StartTs:450115024922083335 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004105200 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115024961404932} {CheckpointTs:450115025354620929 MinTableBarrierTs:450115025354620929 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/364783e3-e914-467f-98e5-8c575a7b7d95 {"id":"364783e3-e914-467f-98e5-8c575a7b7d95","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7c7ff1 364783e3-e914-467f-98e5-8c575a7b7d95 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374684498643733289,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:02:34.965718334+08:00","start-ts":450115024922083335,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115024961404932} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450115025354620929,"min-table-barrier-ts":450115025354620929,"admin-job-type":0} /tidb/cdc/default/default/task/position/364783e3-e914-467f-98e5-8c575a7b7d95/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684498643733289 {"id":7374684498643733289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7374684498643733289 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:02:34.965718334 +0800 CST StartTs:450115024922083335 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004105200 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115024961404932} {CheckpointTs:450115025354620929 MinTableBarrierTs:450115025354620929 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/364783e3-e914-467f-98e5-8c575a7b7d95 {"id":"364783e3-e914-467f-98e5-8c575a7b7d95","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7c7ff1 364783e3-e914-467f-98e5-8c575a7b7d95 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374684498643733289,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:02:34.965718334+08:00","start-ts":450115024922083335,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115024961404932} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450115025354620929,"min-table-barrier-ts":450115025354620929,"admin-job-type":0} /tidb/cdc/default/default/task/position/364783e3-e914-467f-98e5-8c575a7b7d95/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684498643733289 + grep -q 'etcd info' {"id":7374684498643733289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Thu May 30 15:02:50 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.51945196.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:02:50 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7374684498643733289 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:02:34.965718334 +0800 CST StartTs:450115024922083335 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004105200 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115024961404932} {CheckpointTs:450115025354620929 MinTableBarrierTs:450115025354620929 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/364783e3-e914-467f-98e5-8c575a7b7d95 {"id":"364783e3-e914-467f-98e5-8c575a7b7d95","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7c7ff1 364783e3-e914-467f-98e5-8c575a7b7d95 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374684498643733289,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:02:34.965718334+08:00","start-ts":450115024922083335,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115024961404932} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450115025367466020,"min-table-barrier-ts":450115025367466020,"admin-job-type":0} /tidb/cdc/default/default/task/position/364783e3-e914-467f-98e5-8c575a7b7d95/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684498643733289 {"id":7374684498643733289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7374684498643733289 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:02:34.965718334 +0800 CST StartTs:450115024922083335 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004105200 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115024961404932} {CheckpointTs:450115025354620929 MinTableBarrierTs:450115025354620929 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/364783e3-e914-467f-98e5-8c575a7b7d95 {"id":"364783e3-e914-467f-98e5-8c575a7b7d95","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7c7ff1 364783e3-e914-467f-98e5-8c575a7b7d95 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374684498643733289,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:02:34.965718334+08:00","start-ts":450115024922083335,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115024961404932} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450115025367466020,"min-table-barrier-ts":450115025367466020,"admin-job-type":0} /tidb/cdc/default/default/task/position/364783e3-e914-467f-98e5-8c575a7b7d95/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684498643733289 + grep -q 'failed to get info:' {"id":7374684498643733289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7374684498643733289 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:02:34.965718334 +0800 CST StartTs:450115024922083335 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004105200 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115024961404932} {CheckpointTs:450115025354620929 MinTableBarrierTs:450115025354620929 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/364783e3-e914-467f-98e5-8c575a7b7d95 {"id":"364783e3-e914-467f-98e5-8c575a7b7d95","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84e7c7ff1 364783e3-e914-467f-98e5-8c575a7b7d95 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7374684498643733289,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24496?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:02:34.965718334+08:00","start-ts":450115024922083335,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115024961404932} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450115025367466020,"min-table-barrier-ts":450115025367466020,"admin-job-type":0} /tidb/cdc/default/default/task/position/364783e3-e914-467f-98e5-8c575a7b7d95/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684498643733289 {"id":7374684498643733289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later [2024/05/30 15:02:50.906 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:02:50.998 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:02:50.998 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/30 15:02:50.998 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] go: downloading github.com/go-playground/locales v0.14.1 [2024/05/30 15:02:51.099 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/30 15:02:51.102 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/30 15:02:51.294 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:02:51.294 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:02:51.301 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.20.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 wait process cdc.test exit for 3-th time... go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da [2024/05/30 15:02:51.596 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/30 15:02:51.696 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/30 15:02:51.797 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] Verifying downstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:02:51 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 [2024/05/30 15:02:52.397 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/30 15:02:52.403 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] table ddl_manager.finish_mark not exists for 2-th check, retry later [2024/05/30 15:02:53.400 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/30 15:02:53.405 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/30 15:02:53.498 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/30 15:02:53.597 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/30 15:02:53.600 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/30 15:02:53.799 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used [2024/05/30 15:02:53.813 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/30 15:02:53.897 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] table region_merge.t1 exists check diff failed 1-th time, retry later [2024/05/30 15:02:54.600 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6297d663_e5dc_4921_be64_d94fab8dc1d2"] [2024/05/30 15:02:54.608 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs660f49ee_9bdc_419d_a7a5_fcfe3ef610cb"] [2024/05/30 15:02:54.720 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/30 15:02:50.615 +08:00] [INFO] [pd_service_discovery.go:631] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/30 15:02:50.615 +08:00] [INFO] [pd_service_discovery.go:197] ["[pd] init cluster id"] [cluster-id=7374683702768697772] [2024/05/30 15:02:50.615 +08:00] [INFO] [client.go:600] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/30 15:02:50.615 +08:00] [INFO] [tso_client.go:231] ["[tso] switch dc tso global allocator serving address"] [dc-location=global] [new-address=http://127.0.0.1:2379] [2024/05/30 15:02:50.616 +08:00] [INFO] [tso_dispatcher.go:323] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/30 15:02:50.616 +08:00] [INFO] [client.go:648] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/30 15:02:50.617 +08:00] [INFO] [pd_service_discovery.go:631] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/30 15:02:50.617 +08:00] [INFO] [pd_service_discovery.go:197] ["[pd] init cluster id"] [cluster-id=7374683702768697772] [2024/05/30 15:02:50.617 +08:00] [INFO] [client.go:600] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/30 15:02:50.617 +08:00] [INFO] [tso_client.go:231] ["[tso] switch dc tso global allocator serving address"] [dc-location=global] [new-address=http://127.0.0.1:2379] [2024/05/30 15:02:50.618 +08:00] [INFO] [tso_dispatcher.go:323] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/30 15:02:50.618 +08:00] [INFO] [client.go:648] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/30 15:02:50.619 +08:00] [INFO] [tikv_driver.go:200] ["using API V1."] [2024/05/30 15:02:50.620 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/30 15:02:50.620 +08:00] [INFO] [region_cache.go:2683] ["change store resolve state"] [store=2] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] [2024/05/30 15:02:54.910 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/30 15:02:55.006 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/30 15:02:55.202 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs403387e4_bbc3_4ad7_9107_b168606fdc8d"] [2024/05/30 15:02:55.310 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb562a3e6_2009_492f_94fb_523ca362d7c5"] go: downloading github.com/ardielle/ardielle-go v1.5.2 Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:02:55.794 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/30 15:02:55.798 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7324832e_d947_4261_a92f_3811e82ad856"] [2024/05/30 15:02:55.800 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.6 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pingcap/tipb v0.0.0-20240507090649-2bf6bb0cb996 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading golang.org/x/tools v0.21.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec [2024/05/30 15:02:55.996 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:56.003 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:56.003 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:56.010 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] table ddl_manager.finish_mark not exists for 3-th check, retry later go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c [2024/05/30 15:02:56.212 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/05/30 15:02:56.300 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs81953d4d_0bf7_444d_95e6_61d149e6b2e5"] go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 [2024/05/30 15:02:56.697 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:56.701 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs06ba1374_da6c_405f_94c4_0253a76f6ec9"] check diff successfully wait process cdc.test exit for 1-th time... [2024/05/30 15:02:56.796 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:56.910 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:56.911 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] Debugger for raftstore-v2 is used wait process cdc.test exit for 2-th time... [2024/05/30 15:02:57.598 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:57.607 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:57.813 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:02:57.814 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:02:57 CST 2024] <<<<<< run test case region_merge success! >>>>>> table ddl_manager.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:02:57.902 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:02:57.904 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:02:58.102 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] [2024/05/30 15:02:58.103 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] \033[0;36m<<< Run all test success >>>\033[0m [2024/05/30 15:02:58.204 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:58.210 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [Pipeline] } go: downloading github.com/jmespath/go-jmespath v0.4.0 [2024/05/30 15:02:58.397 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs53f68dcb_821e_4bd5_85a1_5284ef12ec5d"] [2024/05/30 15:02:58.505 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:02:58.599 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:02:58.600 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:02:58.610 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [2024/05/30 15:02:58.705 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:02:58.799 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [Pipeline] // cache [Pipeline] } go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/felixge/httpsnoop v1.0.4 [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh: line 1: 10654 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [2024/05/30 15:02:59.712 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [Pipeline] // podTemplate [Pipeline] } table ddl_manager.finish_mark not exists for 5-th check, retry later [Pipeline] // withEnv ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:02:59.910 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:00.010 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [Pipeline] } [Pipeline] // stage [Pipeline] } [2024/05/30 15:03:00.093 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:00.114 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:00.196 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:00.505 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:00.700 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:00.813 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:00.901 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:00.911 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] [2024/05/30 15:03:00.913 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:00.993 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:01.013 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:01.101 +08:00] [INFO] [main.go:178] ["0 insert success: 1400"] [2024/05/30 15:03:01.194 +08:00] [INFO] [main.go:199] ["0 delete success: 700"] [2024/05/30 15:03:01.200 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:01.208 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:00.620 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/30 15:03:00.621 +08:00] [INFO] [tso_dispatcher.go:186] ["exit tso requests cancel loop"] [2024/05/30 15:03:00.621 +08:00] [INFO] [pd_service_discovery.go:248] ["[pd] exit member loop due to context canceled"] [2024/05/30 15:03:00.621 +08:00] [INFO] [tso_dispatcher.go:240] ["exit tso dispatcher loop"] [2024/05/30 15:03:00.621 +08:00] [INFO] [tso_dispatcher.go:410] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/30 15:03:00.621 +08:00] [INFO] [resource_manager_client.go:271] ["[resource manager] exit resource token dispatcher"] [2024/05/30 15:03:00.621 +08:00] [INFO] [tso_dispatcher.go:344] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/05/30 15:03:00.621 +08:00] [INFO] [tso_client.go:134] ["closing tso client"] [2024/05/30 15:03:00.621 +08:00] [INFO] [tso_client.go:139] ["close tso client"] [2024/05/30 15:03:00.621 +08:00] [INFO] [tso_client.go:150] ["tso client is closed"] [2024/05/30 15:03:00.621 +08:00] [INFO] [pd_service_discovery.go:294] ["[pd] close pd service discovery client"] [2024/05/30 15:03:01.411 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:01.611 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table ddl_manager.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd [2024/05/30 15:03:02.200 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:02.297 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_claim_check Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... [2024/05/30 15:03:02.694 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:02.705 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:02.806 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:02.916 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:02.916 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:02.995 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:03.095 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:03.105 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:03.507 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:03.511 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] [2024/05/30 15:03:03.594 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:03.697 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:03.698 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:03.806 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:03.811 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:03.908 +08:00] [INFO] [main.go:178] ["0 insert success: 1500"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cli/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/30 15:03:04.203 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:04.294 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] table ddl_manager.finish_mark not exists for 7-th check, retry later table big_txn.usertable1 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:04.701 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:04.996 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:05.009 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:05.108 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/30 15:03:05.197 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:05.294 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:05.303 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:05.392 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] table ddl_manager.finish_mark not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:06.014 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:06.094 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:06.096 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:06.103 +08:00] [INFO] [main.go:178] ["1 insert success: 2100"] [2024/05/30 15:03:06.294 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:06.302 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:06.304 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:06.308 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:06.503 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:06.710 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:06.896 +08:00] [INFO] [main.go:178] ["0 insert success: 1600"] [2024/05/30 15:03:06.904 +08:00] [INFO] [main.go:199] ["0 delete success: 800"] [2024/05/30 15:03:06.912 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:07.202 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:07.296 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:07.298 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:07.698 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:07.808 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:07.809 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table test.t2 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 9-th check, retry later [2024/05/30 15:03:08.105 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:08.398 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:03:08.500 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:03:08.595 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... [2024/05/30 15:03:08.707 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:08.807 +08:00] [INFO] [main.go:178] ["1 insert success: 2200"] [2024/05/30 15:03:08.810 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:08.893 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:08.900 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table test.t2 not exists for 2-th check, retry later TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/30 15:03:09.104 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:09.194 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:09.393 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:09.398 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:03:09.405 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:09.699 +08:00] [INFO] [main.go:178] ["0 insert success: 1700"] table ddl_manager.finish_mark not exists for 10-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:10.006 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:03:10.013 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:10.294 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:10.297 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:10.299 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:03:10.495 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table test.t2 not exists for 3-th check, retry later Starting Upstream TiDB... [2024/05/30 15:03:10.932 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore [2024/05/30 15:03:11.145 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:11.300 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:11.305 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:11.400 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:11.404 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:11.408 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:03:11.494 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:03:11.494 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:03:11.697 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] table ddl_manager.finish_mark not exists for 11-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:12.102 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:03:12.302 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:12.393 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:12.497 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:12.703 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:12.795 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:12.893 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:03:13.115 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... [2024/05/30 15:03:13.208 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:13.304 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:13.310 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:03:13.312 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:13.398 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table test.t2 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:13.493 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:13.493 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:13.811 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:14.095 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table ddl_manager.finish_mark not exists for 12-th check, retry later [2024/05/30 15:03:14.295 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:03:14.296 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] Verifying downstream PD is started... [2024/05/30 15:03:14.707 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:14.803 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:14.905 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:14.906 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:15.094 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table test.t2 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:15.302 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:15.302 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:03:15.306 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:15.310 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:03:15.409 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:15.506 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:15.602 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:15.707 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:03:15.993 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:16.003 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:16.007 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] table ddl_manager.finish_mark not exists for 13-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:16.409 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:03:16.899 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:17.004 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:17.105 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:17.110 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] table test.t2 not exists for 6-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:17.207 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:17.294 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:17.306 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:03:17.597 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 49.878591193s"] [2024/05/30 15:03:17.603 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:17.697 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:17.802 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 14-th check, retry later [2024/05/30 15:03:17.897 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:17.999 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:03:18.004 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:18.094 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:03:18.116 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:18.313 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:18.592 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table test.t2 not exists for 7-th check, retry later [2024/05/30 15:03:19.000 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:03:19.008 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:03:19.095 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:19.197 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:19.215 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:03:19.304 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:03:19.305 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213d875c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:11631, start at 2024-05-30 15:03:16.717778326 +0800 CST m=+9.808891364 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:16.729 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:16.695 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:16.695 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213d875c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:11631, start at 2024-05-30 15:03:16.717778326 +0800 CST m=+9.808891364 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:16.729 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:16.695 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:16.695 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213d8ce00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:11703, start at 2024-05-30 15:03:17.054124178 +0800 CST m=+9.962669953 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:17.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:17.048 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:17.048 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 15-th check, retry later [2024/05/30 15:03:19.992 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:20.000 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:20.006 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:03:20.009 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:20.101 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:20.298 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:03:20.402 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:20.494 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:03:20.606 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] table test.t2 exists [2024/05/30 15:03:20.916 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:21.002 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:21.008 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] check diff successfully [2024/05/30 15:03:21.096 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:21.398 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213db99c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:9938, start at 2024-05-30 15:03:19.958513906 +0800 CST m=+8.959701919 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:19.974 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:19.962 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:19.962 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213db99c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:9938, start at 2024-05-30 15:03:19.958513906 +0800 CST m=+8.959701919 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:19.974 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:19.962 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:19.962 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213dbb980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:10001, start at 2024-05-30 15:03:20.084861744 +0800 CST m=+8.798727618 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:20.095 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:20.089 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:20.089 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 16-th check, retry later wait process cdc.test exit for 2-th time... [2024/05/30 15:03:21.807 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:21.903 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:03:21.906 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:03:22.000 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Thu May 30 15:03:21 CST 2024] <<<<<< START cdc server in kafka_simple_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.1299712999.out server --log-file /tmp/tidb_cdc_test/kafka_simple_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_claim_check/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/30 15:03:22.218 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:03:22.369 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:03:22.409 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] wait process cdc.test exit for 3-th time... [2024/05/30 15:03:22.696 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/30 15:03:22.708 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/30 15:03:22.800 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:22.900 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:03:23 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> [2024/05/30 15:03:23.204 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:03:23.496 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/30 15:03:23.698 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:23.811 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:03:23.813 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213ddf240006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3, pid:6327, start at 2024-05-30 15:03:22.324446986 +0800 CST m=+26.617134429 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:22.336 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:22.313 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:22.313 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213ddf240006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3, pid:6327, start at 2024-05-30 15:03:22.324446986 +0800 CST m=+26.617134429 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:22.336 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:22.313 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:22.313 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213cc0780009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-bzjr3, pid:6388, start at 2024-05-30 15:03:03.981316858 +0800 CST m=+8.181908587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:03.990 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:03.966 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:03.966 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/30 15:03:24.104 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/30 15:03:24.105 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11216.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/30 15:03:24.600 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:24.706 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/30 15:03:24.814 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:03:24 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1cbf6f48-e5a8-42d7-b730-df9021398317 {"id":"1cbf6f48-e5a8-42d7-b730-df9021398317","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84f2987cd 1cbf6f48-e5a8-42d7-b730-df9021398317 /tidb/cdc/default/default/upstream/7374684693195743751 {"id":7374684693195743751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1cbf6f48-e5a8-42d7-b730-df9021398317 {"id":"1cbf6f48-e5a8-42d7-b730-df9021398317","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84f2987cd 1cbf6f48-e5a8-42d7-b730-df9021398317 /tidb/cdc/default/default/upstream/7374684693195743751 {"id":7374684693195743751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1cbf6f48-e5a8-42d7-b730-df9021398317 {"id":"1cbf6f48-e5a8-42d7-b730-df9021398317","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84f2987cd 1cbf6f48-e5a8-42d7-b730-df9021398317 /tidb/cdc/default/default/upstream/7374684693195743751 {"id":7374684693195743751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.13046.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 17-th check, retry later [2024/05/30 15:03:25.008 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/30 15:03:25.016 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/30 15:03:25.019 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc889d213_5a30_4b7a_bec6_76a92b9d7a2f"] [2024/05/30 15:03:25.103 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs80fa98f8_a3b4_4945_8d92_cc35a8a4670c"] + set +x + tso='450115037839228929 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115037839228929 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2024/05/30 15:03:25.512 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/30 15:03:25.615 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/30 15:03:25.701 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf2209b0b_d16c_43c9_b246_70391bdfb3c5"] [2024/05/30 15:03:25.794 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8b694c02_1d68_41ef_bd73_c60f68e68c82"] [2024/05/30 15:03:25.802 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2044e437_7d97_460a_bebb_7f97b130867d"] [2024/05/30 15:03:25.809 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsfab9a239_2d17_4fc5_86e1_d4f031c49851"] [2024/05/30 15:03:25.911 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:26.006 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:26.007 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:26.208 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213e13800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn, pid:6162, start at 2024-05-30 15:03:25.689375779 +0800 CST m=+5.435626542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:25.696 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:25.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:25.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Thu May 30 15:03:26 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.1933819340.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Thu May 30 15:03:26 CST 2024] <<<<<< START cdc server in cli case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.1126511267.out server --log-file /tmp/tidb_cdc_test/cli/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='450115038148296706 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115038148296706 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.13072.out cli changefeed create --start-ts=450115038148296706 '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-16580?protocol=simple' -c kafka-simple-claim-check --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/conf/changefeed.toml [2024/05/30 15:03:27.094 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:27.112 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:27.196 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc9a462a9_b4c8_43be_b0dd_142ea2ffdca0"] [2024/05/30 15:03:27.198 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] Create changefeed successfully! ID: kafka-simple-claim-check Info: {"upstream_id":7374684693195743751,"namespace":"default","id":"kafka-simple-claim-check","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-16580?protocol=simple","create_time":"2024-05-30T15:03:27.252931007+08:00","start_ts":450115038148296706,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115038148296706,"checkpoint_ts":450115038148296706,"checkpoint_time":"2024-05-30 15:03:25.241"} PASS [2024/05/30 15:03:27.201 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:27.202 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:27.305 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:27.394 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:27.406 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:27.412 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 18-th check, retry later [2024/05/30 15:03:27.503 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:27.608 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:27.822 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213e13800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn, pid:6162, start at 2024-05-30 15:03:25.689375779 +0800 CST m=+5.435626542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:25.696 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:25.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:25.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213e142c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-l9gqn, pid:6296, start at 2024-05-30 15:03:25.756061615 +0800 CST m=+5.339265636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:25.766 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:25.757 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:25.757 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/30 15:03:28.515 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:28.608 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:28.612 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:28.613 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] + set +x [2024/05/30 15:03:28.695 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:28.808 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:28.808 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:28.810 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:28.900 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:28.907 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:28.911 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:28.999 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:29.016 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:29.017 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:03:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41cc8645-b1c4-435e-8a19-529c55b4dac3 {"id":"41cc8645-b1c4-435e-8a19-529c55b4dac3","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ef9c9fb 41cc8645-b1c4-435e-8a19-529c55b4dac3 /tidb/cdc/default/default/upstream/7374684636742359035 {"id":7374684636742359035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41cc8645-b1c4-435e-8a19-529c55b4dac3 {"id":"41cc8645-b1c4-435e-8a19-529c55b4dac3","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ef9c9fb 41cc8645-b1c4-435e-8a19-529c55b4dac3 /tidb/cdc/default/default/upstream/7374684636742359035 {"id":7374684636742359035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41cc8645-b1c4-435e-8a19-529c55b4dac3 {"id":"41cc8645-b1c4-435e-8a19-529c55b4dac3","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ef9c9fb 41cc8645-b1c4-435e-8a19-529c55b4dac3 /tidb/cdc/default/default/upstream/7374684636742359035 {"id":7374684636742359035,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.19606.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-rocks-test-1748?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' table ddl_manager.finish_mark not exists for 19-th check, retry later [2024/05/30 15:03:29.699 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] Create changefeed successfully! ID: bbb06dd9-bbaa-421a-aed1-79803618e6e5 Info: {"upstream_id":7374684636742359035,"namespace":"default","id":"bbb06dd9-bbaa-421a-aed1-79803618e6e5","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-rocks-test-1748?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:03:29.874918553+08:00","start_ts":450115039307497476,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115039307497476,"checkpoint_ts":450115039307497476,"checkpoint_time":"2024-05-30 15:03:29.663"} PASS [2024/05/30 15:03:29.793 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:29.802 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:29.802 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:29.807 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:29.911 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:03:29 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df6ee368-983b-443d-8e5e-c3cbe1d6bc8b {"id":"df6ee368-983b-443d-8e5e-c3cbe1d6bc8b","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84f3ad347 df6ee368-983b-443d-8e5e-c3cbe1d6bc8b /tidb/cdc/default/default/upstream/7374684707635413799 {"id":7374684707635413799,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df6ee368-983b-443d-8e5e-c3cbe1d6bc8b {"id":"df6ee368-983b-443d-8e5e-c3cbe1d6bc8b","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84f3ad347 df6ee368-983b-443d-8e5e-c3cbe1d6bc8b /tidb/cdc/default/default/upstream/7374684707635413799 {"id":7374684707635413799,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df6ee368-983b-443d-8e5e-c3cbe1d6bc8b {"id":"df6ee368-983b-443d-8e5e-c3cbe1d6bc8b","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84f3ad347 df6ee368-983b-443d-8e5e-c3cbe1d6bc8b /tidb/cdc/default/default/upstream/7374684707635413799 {"id":7374684707635413799,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11318.out cli changefeed create --start-ts=450115037839228929 '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-8781?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. [2024/05/30 15:03:30.197 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:30.205 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Create changefeed successfully! ID: custom-changefeed-name [2024/05/30 15:03:30.296 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:30.316 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:30.399 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:30.410 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:30.412 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:30.413 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9da3798c_afaa_4b1b_bff0_96b35664df33"] [2024/05/30 15:03:30.516 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] Info: {"upstream_id":7374684707635413799,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-8781?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:03:30.276833962+08:00","start_ts":450115037839228929,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115037839228929,"checkpoint_ts":450115037839228929,"checkpoint_time":"2024-05-30 15:03:24.062"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.7522.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 20-th check, retry later [2024/05/30 15:03:31.104 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:31.202 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:31.303 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:31.305 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:31.305 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:31.410 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + set +x [Thu May 30 15:03:31 CST 2024] <<<<<< START kafka consumer in multi_rocks case >>>>>> + set +x [Thu May 30 15:03:31 CST 2024] <<<<<< START kafka consumer in cli case >>>>>> table test.simple not exists for 1-th check, retry later [2024/05/30 15:03:31.794 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:31.810 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:31.897 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:31.920 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:31.993 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:31.997 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:32.014 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:32.022 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:32.107 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:32.119 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:32.412 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:32.496 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] ***************** properties ***************** "threadcount"="2" "recordcount"="1000" "insertproportion"="0" "mysql.host"="127.0.0.1" "readproportion"="0" "workload"="core" "operationcount"="0" "scanproportion"="0" "mysql.user"="root" "mysql.port"="4000" "mysql.db"="multi_rocks" "requestdistribution"="uniform" "dotransactions"="false" "readallfields"="true" "updateproportion"="0" "table"="a1" ********************************************** + set +x + tso='450115039661654017 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115039661654017 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:03:32 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.75507552.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info [2024/05/30 15:03:32.610 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:32.610 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:32.698 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/30 15:03:32.797 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:33.107 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:33.207 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] table ddl_manager.finish_mark not exists for 21-th check, retry later Run finished, takes 697.225555ms INSERT - Takes(s): 0.7, Count: 1000, OPS: 1477.4, Avg(us): 1359, Min(us): 898, Max(us): 36604, 95th(us): 2000, 99th(us): 4000 [2024/05/30 15:03:33.292 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:33.311 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:33.398 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:33.404 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:33.415 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:33.506 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:33.515 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:33.602 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] ***************** properties ***************** "recordcount"="1000" "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.user"="root" "mysql.db"="multi_rocks" "scanproportion"="0" "operationcount"="0" "requestdistribution"="uniform" "table"="a2" "updateproportion"="0" "mysql.port"="4000" "insertproportion"="0" "workload"="core" "threadcount"="2" "readproportion"="0" "dotransactions"="false" ********************************************** [2024/05/30 15:03:33.828 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:34.093 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:34.237 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:34.269 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table test.simple not exists for 2-th check, retry later [2024/05/30 15:03:34.297 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:34.314 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] Run finished, takes 777.655175ms INSERT - Takes(s): 0.8, Count: 1000, OPS: 1318.4, Avg(us): 1515, Min(us): 887, Max(us): 59367, 95th(us): 2000, 99th(us): 10000 [2024/05/30 15:03:34.717 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] ***************** properties ***************** "operationcount"="0" "threadcount"="2" "mysql.host"="127.0.0.1" "dotransactions"="false" "recordcount"="1000" "scanproportion"="0" "readproportion"="0" "insertproportion"="0" "readallfields"="true" "mysql.user"="root" "mysql.db"="multi_rocks" "mysql.port"="4000" "updateproportion"="0" "workload"="core" "requestdistribution"="uniform" "table"="a3" ********************************************** table ddl_manager.finish_mark not exists for 22-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/30 15:03:35.104 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:35.194 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:35.199 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:35.212 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:35.214 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:35.300 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:35.304 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:35.310 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.13121.out cli changefeed pause -c kafka-simple-claim-check [2024/05/30 15:03:35.410 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:35.508 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:03:35.612 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:03:35.796 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:03:35.805 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:35.805 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:03:35.812 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:03:35 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9172ddd4-7ea7-4ed2-ac5f-b588ee6fe100 {"id":"9172ddd4-7ea7-4ed2-ac5f-b588ee6fe100","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84f53a4ca 9172ddd4-7ea7-4ed2-ac5f-b588ee6fe100 /tidb/cdc/default/default/upstream/7374684735802846293 {"id":7374684735802846293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9172ddd4-7ea7-4ed2-ac5f-b588ee6fe100 {"id":"9172ddd4-7ea7-4ed2-ac5f-b588ee6fe100","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84f53a4ca 9172ddd4-7ea7-4ed2-ac5f-b588ee6fe100 /tidb/cdc/default/default/upstream/7374684735802846293 {"id":7374684735802846293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9172ddd4-7ea7-4ed2-ac5f-b588ee6fe100 {"id":"9172ddd4-7ea7-4ed2-ac5f-b588ee6fe100","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84f53a4ca 9172ddd4-7ea7-4ed2-ac5f-b588ee6fe100 /tidb/cdc/default/default/upstream/7374684735802846293 {"id":7374684735802846293,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.7598.out cli changefeed create --start-ts=450115039661654017 '--sink-uri=kafka://127.0.0.1:9092/ticdc-common-1-test-26508?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... Run finished, takes 690.255336ms INSERT - Takes(s): 0.7, Count: 1000, OPS: 1492.8, Avg(us): 1343, Min(us): 945, Max(us): 20315, 95th(us): 2000, 99th(us): 3000 table test.simple exists table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450115038376362015, "checkpoint_time": "2024-05-30 15:03:26.111", "error": null }' + echo '{ "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450115038376362015, "checkpoint_time": "2024-05-30 15:03:26.111", "error": null }' { "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450115038376362015, "checkpoint_time": "2024-05-30 15:03:26.111", "error": null } ++ echo '{' '"upstream_id":' 7374684707635413799, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450115038376362015, '"checkpoint_time":' '"2024-05-30' '15:03:26.111",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374684707635413799, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450115038376362015, '"checkpoint_time":' '"2024-05-30' '15:03:26.111",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 ***************** properties ***************** "insertproportion"="0" "scanproportion"="0" "mysql.user"="root" "table"="a4" "readproportion"="0" "mysql.port"="4000" "dotransactions"="false" "recordcount"="1000" "readallfields"="true" "operationcount"="0" "mysql.host"="127.0.0.1" "threadcount"="2" "requestdistribution"="uniform" "workload"="core" "mysql.db"="multi_rocks" "updateproportion"="0" ********************************************** [2024/05/30 15:03:36.408 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 [2024/05/30 15:03:36.592 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:36.596 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:36.609 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 [2024/05/30 15:03:36.709 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:36.793 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:36.802 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:36.809 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:36.895 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:36.909 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:36.910 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:03:37.102 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Create changefeed successfully! ID: a08d656e-a219-4e07-843c-8393f435120a Info: {"upstream_id":7374684735802846293,"namespace":"default","id":"a08d656e-a219-4e07-843c-8393f435120a","sink_uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-26508?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:03:37.020024937+08:00","start_ts":450115039661654017,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115039661654017,"checkpoint_ts":450115039661654017,"checkpoint_time":"2024-05-30 15:03:31.014"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.13144.out cli changefeed update -c kafka-simple-claim-check '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-16580?protocol=simple&max-message-bytes=2048' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/conf/changefeed.toml --no-confirm [2024/05/30 15:03:37.301 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:03:37.314 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:03:37.315 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:03:37.397 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 23-th check, retry later Run finished, takes 1.030555375s INSERT - Takes(s): 1.0, Count: 1000, OPS: 998.3, Avg(us): 2019, Min(us): 864, Max(us): 45903, 95th(us): 3000, 99th(us): 29000 Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc0031dcd18} {Type:update Path:[Config SyncPointRetention] From: To:0xc0031dcd30} {Type:update Path:[Config Consistent] From: To:0xc0009a24d0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11613.out cli changefeed --changefeed-id custom-changefeed-name pause ***************** properties ***************** "updateproportion"="0" "mysql.user"="root" "recordcount"="1000" "insertproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "requestdistribution"="uniform" "mysql.db"="multi_rocks" "dotransactions"="false" "readallfields"="true" "workload"="core" "table"="a5" "readproportion"="0" "threadcount"="2" "scanproportion"="0" ********************************************** Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/kafka-simple-claim-check-16580?protocol=simple To:kafka://127.0.0.1:9092/kafka-simple-claim-check-16580?protocol=simple&max-message-bytes=2048} {Type:update Path:[Config SyncPointInterval] From: To:0xc0034d0fb8} {Type:update Path:[Config SyncPointRetention] From: To:0xc0034d0fc8} {Type:update Path:[Config Consistent] From: To:0xc0009c8150} Update changefeed config successfully! ID: kafka-simple-claim-check Info: {"upstream_id":7374684693195743751,"namespace":"default","id":"kafka-simple-claim-check","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-16580?protocol=simple\u0026max-message-bytes=2048","create_time":"2024-05-30T15:03:27.252931007+08:00","start_ts":450115038148296706,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115040731201543,"checkpoint_ts":450115040731201543,"checkpoint_time":"2024-05-30 15:03:35.094"} PASS [2024/05/30 15:03:37.802 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:03:38.095 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:38.098 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:38.202 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:03:38.395 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:38.413 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:38.500 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:38.509 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:38.512 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] + set +x [Thu May 30 15:03:38 CST 2024] <<<<<< START kafka consumer in common_1 case >>>>>> [2024/05/30 15:03:38.515 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:38.597 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:38.615 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:38.902 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:39.011 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:03:39.011 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:39.101 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] Run finished, takes 1.134690318s INSERT - Takes(s): 1.1, Count: 1000, OPS: 898.5, Avg(us): 2185, Min(us): 885, Max(us): 76975, 95th(us): 3000, 99th(us): 41000 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 24-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.13168.out cli changefeed resume -c kafka-simple-claim-check + set +x [2024/05/30 15:03:39.593 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:03:39.901 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:39.993 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:40.000 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:40.010 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] skip some SQLs in tidb v4.0.x table common_1.v1 not exists for 1-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:03:40.107 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:40.197 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:40.302 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:40.307 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:03:40.309 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:40.313 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:03:40.314 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:40.403 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table multi_rocks.finish_mark not exists for 1-th check, retry later [2024/05/30 15:03:40.698 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:03:40.703 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:40.707 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:41.095 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:41.311 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table ddl_manager.finish_mark not exists for 25-th check, retry later + set +x TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:03:39 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> [2024/05/30 15:03:41.515 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:41.515 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:41.600 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:41.608 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:41.611 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:41.613 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:41.709 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:41.710 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:41.797 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:03:41.805 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:03:41.809 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 1-th check, retry later [2024/05/30 15:03:42.094 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:03:42.098 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:03:42.100 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] table common_1.v1 not exists for 2-th check, retry later table multi_rocks.finish_mark not exists for 2-th check, retry later [2024/05/30 15:03:42.592 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:03:42.816 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450115041259945992, "checkpoint_time": "2024-05-30 15:03:37.111", "error": null }' + echo '{ "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450115041259945992, "checkpoint_time": "2024-05-30 15:03:37.111", "error": null }' { "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450115041259945992, "checkpoint_time": "2024-05-30 15:03:37.111", "error": null } ++ echo '{' '"upstream_id":' 7374684707635413799, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 450115041259945992, '"checkpoint_time":' '"2024-05-30' '15:03:37.111",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7374684707635413799, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 450115041259945992, '"checkpoint_time":' '"2024-05-30' '15:03:37.111",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11670.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name [2024/05/30 15:03:43.102 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:43.112 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:03:43.114 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table ddl_manager.finish_mark not exists for 26-th check, retry later [2024/05/30 15:03:43.196 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:03:43.198 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:03:43.206 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:03:43.209 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:03:43.304 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:03:43.316 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:03:43.317 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:03:43.411 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc0038ff028} {Type:update Path:[Config SyncPointRetention] From: To:0xc0038ff038} {Type:update Path:[Config Consistent] From: To:0xc000d8e620} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} [2024/05/30 15:03:43.495 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7374684707635413799,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-8781?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:03:30.276833962+08:00","start_ts":450115037839228929,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":true,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115041259945992,"checkpoint_ts":450115041259945992,"checkpoint_time":"2024-05-30 15:03:37.111"} PASS coverage: 2.8% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:43.896 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later [2024/05/30 15:03:44.210 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:03:44.504 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table test.finish_mark not exists for 2-th check, retry later [2024/05/30 15:03:44.515 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:44.520 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:03:44.597 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:03:44.598 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:03:44.620 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:44.626 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:44.626 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:03:44.627 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] table multi_rocks.finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 27-th check, retry later [2024/05/30 15:03:45.098 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11709.out cli changefeed --changefeed-id custom-changefeed-name resume ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:45.808 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [Thu May 30 15:03:39 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 1 [Thu May 30 15:03:39 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 2 [Thu May 30 15:03:39 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 3 table test.table1 not exists for 1-th check, retry later table test.table1 not exists for 2-th check, retry later table test.table1 exists table test.table2 exists table test.table3 exists check diff successfully table test.table10 not exists for 1-th check, retry later table test.table10 exists table test.table20 exists check diff successfully [2024/05/30 15:03:45.901 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/30 15:03:45.910 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/30 15:03:45.911 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:45.923 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:03:46.025 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table test.finish_mark exists check diff successfully [2024/05/30 15:03:46.398 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5c0dcdd7_e08e_4e2f_b3dc_41313c960114"] [2024/05/30 15:03:46.411 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/30 15:03:46.620 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] wait process cdc.test exit for 1-th time... [2024/05/30 15:03:46.711 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/30 15:03:46.712 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/30 15:03:46.716 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:03:46.903 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] table multi_rocks.finish_mark exists check diff successfully [2024/05/30 15:03:47.100 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsabb93be3_979c_4c77_97b0_4efdcc42ec80"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... [2024/05/30 15:03:47.209 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:47.308 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:47.321 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213f64840013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:22017, start at 2024-05-30 15:03:47.262511595 +0800 CST m=+5.846541516 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:47.269 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:47.233 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:47.233 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:47.508 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs22d231f3_18aa_4f39_b6cf_5b7342798b41"] [2024/05/30 15:03:47.605 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb6a07a0d_cf18_494e_a87c_192feefa558e"] table ddl_manager.finish_mark not exists for 28-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... [2024/05/30 15:03:47.797 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:03:47.909 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:03:47.914 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5769cd8a_1f02_4a71_9760_429b17d59f53"] [2024/05/30 15:03:48.201 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] wait process cdc.test exit for 3-th time... [2024/05/30 15:03:48.212 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:48.309 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:48.412 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:03:48 CST 2024] <<<<<< run test case kafka_simple_claim_check success! >>>>>> table common_1.recover_and_insert not exists for 2-th check, retry later [2024/05/30 15:03:48.703 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:48.704 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:48.803 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:48.805 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:03:48 CST 2024] <<<<<< run test case multi_rocks success! >>>>>> table ddl_manager.finish_mark not exists for 29-th check, retry later [2024/05/30 15:03:49.596 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7cf73ea4_e022_49ec_a540_c4cba206c2ee"] [2024/05/30 15:03:49.693 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:49.695 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:49.795 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa11134cd_5e71_4464_8dca_57c11ee518db"] [2024/05/30 15:03:49.905 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:50.006 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:50.110 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213f64840013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:22017, start at 2024-05-30 15:03:47.262511595 +0800 CST m=+5.846541516 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:47.269 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:47.233 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:47.233 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f213f66080014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:22096, start at 2024-05-30 15:03:47.355524482 +0800 CST m=+5.857418259 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:05:47.362 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:03:47.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:53:47.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/30 15:03:50.316 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450115044667817990, "checkpoint_time": "2024-05-30 15:03:50.111", "error": null }' + echo '{ "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450115044667817990, "checkpoint_time": "2024-05-30 15:03:50.111", "error": null }' { "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450115044667817990, "checkpoint_time": "2024-05-30 15:03:50.111", "error": null } ++ echo '{' '"upstream_id":' 7374684707635413799, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450115044667817990, '"checkpoint_time":' '"2024-05-30' '15:03:50.111",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374684707635413799, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450115044667817990, '"checkpoint_time":' '"2024-05-30' '15:03:50.111",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11780.out cli changefeed --changefeed-id custom-changefeed-name remove [2024/05/30 15:03:50.699 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:50.819 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:50.939 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:51.040 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:51.300 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:51.311 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:51.394 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:51.409 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:51.412 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 450115044667817990 SinkURI: kafka://127.0.0.1:9092/ticdc-cli-test-8781?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS table ddl_manager.finish_mark not exists for 30-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/30 15:03:51.497 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:51.503 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:03:51.798 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:51.799 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.23368.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/30 15:03:52.003 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:52.096 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:52.199 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:52.307 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:52.596 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:52.800 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:52.808 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb7ae0c38_f9fc_47f1_b50d_7c70198cb996"] [2024/05/30 15:03:52.812 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:52.907 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:03:52.908 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table common_1.finish_mark not exists for 2-th check, retry later + set +x [2024/05/30 15:03:52.995 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:53.003 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:53.006 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table ddl_manager.finish_mark not exists for 31-th check, retry later [2024/05/30 15:03:53.393 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:53.399 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] + set +x + tso='450115045210980353 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115045210980353 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "workload"="core" "updateproportion"="0" "mysql.port"="4000" "recordcount"="10000" "scanproportion"="0" "mysql.db"="move_table" "mysql.user"="root" "threadcount"="10" "readallfields"="true" "requestdistribution"="uniform" "operationcount"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" "dotransactions"="false" ********************************************** [2024/05/30 15:03:53.804 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:53.809 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:54.094 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:54.295 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:54.405 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:54.513 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... [2024/05/30 15:03:54.610 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:54.611 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:03:54.704 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:03:54.705 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:54.798 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:54.804 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:54.806 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] table common_1.finish_mark not exists for 3-th check, retry later [2024/05/30 15:03:54.893 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:55.199 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:55.204 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:55.504 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:55.595 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] Verifying downstream PD is started... Starting Upstream TiKV... [2024/05/30 15:03:55.703 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] table ddl_manager.finish_mark not exists for 32-th check, retry later TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/30 15:03:55.997 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11837.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-8781?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [2024/05/30 15:03:56.196 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:56.298 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:03:56.314 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7374684707635413799,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-8781?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:03:56.596428022+08:00","start_ts":450115046332432388,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115046332432388,"checkpoint_ts":450115046332432388,"checkpoint_time":"2024-05-30 15:03:56.461"} PASS [2024/05/30 15:03:56.460 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:56.496 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] table common_1.finish_mark exists check diff successfully coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:03:56.902 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:56.909 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:03:57.006 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:03:57.011 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:03:57.104 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] Run finished, takes 3.137945809s INSERT - Takes(s): 3.1, Count: 10000, OPS: 3192.8, Avg(us): 3097, Min(us): 705, Max(us): 78185, 95th(us): 4000, 99th(us): 58000 [Thu May 30 15:03:56 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2345523457.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... [2024/05/30 15:03:57.194 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:03:57.328 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table ddl_manager.finish_mark not exists for 33-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... wait process cdc.test exit for 2-th time... [2024/05/30 15:03:57.832 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [2024/05/30 15:03:57.993 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] + set +x [Thu May 30 15:03:58 CST 2024] <<<<<< run test case common_1 success! >>>>>> [2024/05/30 15:03:58.302 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:58.946 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:59.238 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:03:59.335 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:03:59.409 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:03:59.428 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:03:59.561 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] table ddl_manager.finish_mark not exists for 34-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:03:59.898 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:03:59.905 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:04:00.011 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:04:00.097 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:04:00.106 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:04:00.201 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:04:00.296 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:04:00.410 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:04:00.506 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:04:00.610 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:04:00 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45445e04-8969-4898-b0b2-509da30839a0 {"id":"45445e04-8969-4898-b0b2-509da30839a0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb26fe6 45445e04-8969-4898-b0b2-509da30839a0 /tidb/cdc/default/default/upstream/7374684843072506872 {"id":7374684843072506872,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45445e04-8969-4898-b0b2-509da30839a0 {"id":"45445e04-8969-4898-b0b2-509da30839a0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb26fe6 45445e04-8969-4898-b0b2-509da30839a0 /tidb/cdc/default/default/upstream/7374684843072506872 {"id":7374684843072506872,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45445e04-8969-4898-b0b2-509da30839a0 {"id":"45445e04-8969-4898-b0b2-509da30839a0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb26fe6 45445e04-8969-4898-b0b2-509da30839a0 /tidb/cdc/default/default/upstream/7374684843072506872 {"id":7374684843072506872,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.23505.out cli changefeed create --start-ts=450115045210980353 '--sink-uri=kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh: line 1: 13192 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/30 15:04:01.012 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:04:01.094 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:04:01.096 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:04:01.096 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:04:01.111 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:04:01.114 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] \033[0;36m<<< Run all test success >>>\033[0m Create changefeed successfully! ID: 8ffdd20c-58fb-4892-9cd5-d34fe4050a0a Info: {"upstream_id":7374684843072506872,"namespace":"default","id":"8ffdd20c-58fb-4892-9cd5-d34fe4050a0a","sink_uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:04:00.873141461+08:00","start_ts":450115045210980353,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115045210980353,"checkpoint_ts":450115045210980353,"checkpoint_time":"2024-05-30 15:03:52.183"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [2024/05/30 15:04:01.413 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [Pipeline] // dir [Pipeline] } + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450115047289257991, "checkpoint_time": "2024-05-30 15:04:00.111", "error": null }' + echo '{ "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450115047289257991, "checkpoint_time": "2024-05-30 15:04:00.111", "error": null }' { "upstream_id": 7374684707635413799, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450115047289257991, "checkpoint_time": "2024-05-30 15:04:00.111", "error": null } ++ echo '{' '"upstream_id":' 7374684707635413799, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450115047289257991, '"checkpoint_time":' '"2024-05-30' '15:04:00.111",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374684707635413799, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450115047289257991, '"checkpoint_time":' '"2024-05-30' '15:04:00.111",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] [Pipeline] // withCredentials [Pipeline] } [2024/05/30 15:04:01.594 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:04:01.598 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [2024/05/30 15:04:01.811 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:04:01.898 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:04:01.917 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [Pipeline] } [Pipeline] // withEnv [Pipeline] } + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11935.out cli changefeed create --start-ts=450115037839228929 '--sink-uri=kafka://127.0.0.1:9093/ticdc-cli-test-ssl-5972?protocol=open-protocol&ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem&cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem&key=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem&kafka-version=2.4.1&max-message-bytes=10485760&insecure-skip-verify=true' --tz=Asia/Shanghai table ddl_manager.finish_mark not exists for 35-th check, retry later [Pipeline] // node [2024/05/30 15:04:01.992 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:04:01.993 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:04:02.107 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [Pipeline] } [Pipeline] // podTemplate [WARN] --tz is deprecated in changefeed settings. [Pipeline] } [2024/05/30 15:04:02.308 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [Pipeline] // withEnv [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Thu May 30 15:04:02 CST 2024] <<<<<< START kafka consumer in move_table case >>>>>> [Thu May 30 15:04:02 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2354023542.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // stage [Pipeline] } Create changefeed successfully! ID: 93a33774-837f-45f6-99a9-4773fe8cec4d Info: {"upstream_id":7374684707635413799,"namespace":"default","id":"93a33774-837f-45f6-99a9-4773fe8cec4d","sink_uri":"kafka://127.0.0.1:9093/ticdc-cli-test-ssl-5972?protocol=open-protocol\u0026ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem\u0026cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem\u0026key=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760\u0026insecure-skip-verify=true","create_time":"2024-05-30T15:04:02.716789906+08:00","start_ts":450115037839228929,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115037839228929,"checkpoint_ts":450115037839228929,"checkpoint_time":"2024-05-30 15:03:24.062"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:04:03.002 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:04:03.008 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:04:03.014 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:04:03.092 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:04:03.098 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:04:03.111 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:04:03.293 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:04:03.400 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:04:03.606 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_claim_check_avro Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 36-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11966.out cli unsafe delete-service-gc-safepoint ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:04:05.119 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:04:05.129 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:04:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {UpstreamID:7374684843072506872 Namespace:default ID:8ffdd20c-58fb-4892-9cd5-d34fe4050a0a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:00.873141461 +0800 CST StartTs:450115045210980353 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036fecf0 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115047478525957} {CheckpointTs:450115045702238236 MinTableBarrierTs:450115048461565957 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45445e04-8969-4898-b0b2-509da30839a0 {"id":"45445e04-8969-4898-b0b2-509da30839a0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/6f6a055e-cb62-4382-95d5-9df8e1b9f803 {"id":"6f6a055e-cb62-4382-95d5-9df8e1b9f803","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb26fe6 45445e04-8969-4898-b0b2-509da30839a0 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb27039 6f6a055e-cb62-4382-95d5-9df8e1b9f803 /tidb/cdc/default/default/changefeed/info/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"upstream-id":7374684843072506872,"namespace":"default","changefeed-id":"8ffdd20c-58fb-4892-9cd5-d34fe4050a0a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:00.873141461+08:00","start-ts":450115045210980353,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115047478525957} /tidb/cdc/default/default/changefeed/status/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":450115045702238236,"min-table-barrier-ts":450115048461565957,"admin-job-type":0} /tidb/cdc/default/default/task/position/45445e04-8969-4898-b0b2-509da30839a0/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f6a055e-cb62-4382-95d5-9df8e1b9f803/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684843072506872 {"id":7374684843072506872,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {UpstreamID:7374684843072506872 Namespace:default ID:8ffdd20c-58fb-4892-9cd5-d34fe4050a0a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:00.873141461 +0800 CST StartTs:450115045210980353 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036fecf0 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115047478525957} {CheckpointTs:450115045702238236 MinTableBarrierTs:450115048461565957 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45445e04-8969-4898-b0b2-509da30839a0 {"id":"45445e04-8969-4898-b0b2-509da30839a0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/6f6a055e-cb62-4382-95d5-9df8e1b9f803 {"id":"6f6a055e-cb62-4382-95d5-9df8e1b9f803","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb26fe6 45445e04-8969-4898-b0b2-509da30839a0 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb27039 6f6a055e-cb62-4382-95d5-9df8e1b9f803 /tidb/cdc/default/default/changefeed/info/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"upstream-id":7374684843072506872,"namespace":"default","changefeed-id":"8ffdd20c-58fb-4892-9cd5-d34fe4050a0a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:00.873141461+08:00","start-ts":450115045210980353,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115047478525957} /tidb/cdc/default/default/changefeed/status/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":450115045702238236,"min-table-barrier-ts":450115048461565957,"admin-job-type":0} /tidb/cdc/default/default/task/position/45445e04-8969-4898-b0b2-509da30839a0/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f6a055e-cb62-4382-95d5-9df8e1b9f803/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684843072506872 {"id":7374684843072506872,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {UpstreamID:7374684843072506872 Namespace:default ID:8ffdd20c-58fb-4892-9cd5-d34fe4050a0a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:00.873141461 +0800 CST StartTs:450115045210980353 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036fecf0 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115047478525957} {CheckpointTs:450115045702238236 MinTableBarrierTs:450115048461565957 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45445e04-8969-4898-b0b2-509da30839a0 {"id":"45445e04-8969-4898-b0b2-509da30839a0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/6f6a055e-cb62-4382-95d5-9df8e1b9f803 {"id":"6f6a055e-cb62-4382-95d5-9df8e1b9f803","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb26fe6 45445e04-8969-4898-b0b2-509da30839a0 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb27039 6f6a055e-cb62-4382-95d5-9df8e1b9f803 /tidb/cdc/default/default/changefeed/info/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"upstream-id":7374684843072506872,"namespace":"default","changefeed-id":"8ffdd20c-58fb-4892-9cd5-d34fe4050a0a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:00.873141461+08:00","start-ts":450115045210980353,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115047478525957} /tidb/cdc/default/default/changefeed/status/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":450115045702238236,"min-table-barrier-ts":450115048461565957,"admin-job-type":0} /tidb/cdc/default/default/task/position/45445e04-8969-4898-b0b2-509da30839a0/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f6a055e-cb62-4382-95d5-9df8e1b9f803/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684843072506872 {"id":7374684843072506872,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Thu May 30 15:04:05 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2363723639.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 37-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2140683c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:6285, start at 2024-05-30 15:04:03.878679549 +0800 CST m=+6.066096622 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:06:03.887 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:04:03.855 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:54:03.855 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2140683c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:6285, start at 2024-05-30 15:04:03.878679549 +0800 CST m=+6.066096622 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:06:03.887 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:04:03.855 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:54:03.855 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21406b740006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:6356, start at 2024-05-30 15:04:04.06418916 +0800 CST m=+6.099378565 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:06:04.075 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:04:04.061 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:54:04.061 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11992.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:04:06.398 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:04:06.600 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:04:06.601 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:04:06.698 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/30 15:04:07.704 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table ddl_manager.finish_mark not exists for 38-th check, retry later + set +x [2024/05/30 15:04:07.713 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:04:07.717 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:04:07.898 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:04:07.905 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:04:08.002 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:04:08.007 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:04:08.296 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:04:02.998 +08:00] [WARN] [diff.go:179] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 31"] [2024/05/30 15:04:08.499 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:04:08.606 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:04:08.702 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [Thu May 30 15:04:08 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.75847586.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/30 15:04:08.795 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:04:08.799 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:04:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {UpstreamID:7374684843072506872 Namespace:default ID:8ffdd20c-58fb-4892-9cd5-d34fe4050a0a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:00.873141461 +0800 CST StartTs:450115045210980353 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00358a7e0 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115047478525957} {CheckpointTs:450115046305169422 MinTableBarrierTs:450115049267396610 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36140922-90e6-4ad8-8207-c14da8f49abe {"id":"36140922-90e6-4ad8-8207-c14da8f49abe","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/45445e04-8969-4898-b0b2-509da30839a0 {"id":"45445e04-8969-4898-b0b2-509da30839a0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/6f6a055e-cb62-4382-95d5-9df8e1b9f803 {"id":"6f6a055e-cb62-4382-95d5-9df8e1b9f803","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb26fe6 45445e04-8969-4898-b0b2-509da30839a0 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb27039 6f6a055e-cb62-4382-95d5-9df8e1b9f803 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb27069 36140922-90e6-4ad8-8207-c14da8f49abe /tidb/cdc/default/default/changefeed/info/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"upstream-id":7374684843072506872,"namespace":"default","changefeed-id":"8ffdd20c-58fb-4892-9cd5-d34fe4050a0a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:00.873141461+08:00","start-ts":450115045210980353,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115047478525957} /tidb/cdc/default/default/changefeed/status/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":450115046305169516,"min-table-barrier-ts":450115049267396610,"admin-job-type":0} /tidb/cdc/default/default/task/position/36140922-90e6-4ad8-8207-c14da8f49abe/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/45445e04-8969-4898-b0b2-509da30839a0/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f6a055e-cb62-4382-95d5-9df8e1b9f803/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684843072506872 {"id":7374684843072506872,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {UpstreamID:7374684843072506872 Namespace:default ID:8ffdd20c-58fb-4892-9cd5-d34fe4050a0a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:00.873141461 +0800 CST StartTs:450115045210980353 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00358a7e0 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115047478525957} {CheckpointTs:450115046305169422 MinTableBarrierTs:450115049267396610 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36140922-90e6-4ad8-8207-c14da8f49abe {"id":"36140922-90e6-4ad8-8207-c14da8f49abe","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/45445e04-8969-4898-b0b2-509da30839a0 {"id":"45445e04-8969-4898-b0b2-509da30839a0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/6f6a055e-cb62-4382-95d5-9df8e1b9f803 {"id":"6f6a055e-cb62-4382-95d5-9df8e1b9f803","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb26fe6 45445e04-8969-4898-b0b2-509da30839a0 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb27039 6f6a055e-cb62-4382-95d5-9df8e1b9f803 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb27069 36140922-90e6-4ad8-8207-c14da8f49abe /tidb/cdc/default/default/changefeed/info/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"upstream-id":7374684843072506872,"namespace":"default","changefeed-id":"8ffdd20c-58fb-4892-9cd5-d34fe4050a0a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:00.873141461+08:00","start-ts":450115045210980353,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115047478525957} /tidb/cdc/default/default/changefeed/status/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":450115046305169516,"min-table-barrier-ts":450115049267396610,"admin-job-type":0} /tidb/cdc/default/default/task/position/36140922-90e6-4ad8-8207-c14da8f49abe/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/45445e04-8969-4898-b0b2-509da30839a0/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f6a055e-cb62-4382-95d5-9df8e1b9f803/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684843072506872 {"id":7374684843072506872,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {UpstreamID:7374684843072506872 Namespace:default ID:8ffdd20c-58fb-4892-9cd5-d34fe4050a0a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:00.873141461 +0800 CST StartTs:450115045210980353 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00358a7e0 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115047478525957} {CheckpointTs:450115046305169422 MinTableBarrierTs:450115049267396610 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36140922-90e6-4ad8-8207-c14da8f49abe {"id":"36140922-90e6-4ad8-8207-c14da8f49abe","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/45445e04-8969-4898-b0b2-509da30839a0 {"id":"45445e04-8969-4898-b0b2-509da30839a0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/6f6a055e-cb62-4382-95d5-9df8e1b9f803 {"id":"6f6a055e-cb62-4382-95d5-9df8e1b9f803","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb26fe6 45445e04-8969-4898-b0b2-509da30839a0 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb27039 6f6a055e-cb62-4382-95d5-9df8e1b9f803 /tidb/cdc/default/__cdc_meta__/owner/22318fc84fb27069 36140922-90e6-4ad8-8207-c14da8f49abe /tidb/cdc/default/default/changefeed/info/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"upstream-id":7374684843072506872,"namespace":"default","changefeed-id":"8ffdd20c-58fb-4892-9cd5-d34fe4050a0a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-8368?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:00.873141461+08:00","start-ts":450115045210980353,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115047478525957} /tidb/cdc/default/default/changefeed/status/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":450115046305169516,"min-table-barrier-ts":450115049267396610,"admin-job-type":0} /tidb/cdc/default/default/task/position/36140922-90e6-4ad8-8207-c14da8f49abe/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/45445e04-8969-4898-b0b2-509da30839a0/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f6a055e-cb62-4382-95d5-9df8e1b9f803/8ffdd20c-58fb-4892-9cd5-d34fe4050a0a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684843072506872 {"id":7374684843072506872,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table move_table.usertable exists go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 [2024/05/30 15:04:09.399 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:04:09.399 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:04:09.408 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:04:09.505 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240507075657-2bd4573edde2 [2024/05/30 15:04:09.604 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:04:09.709 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 [2024/05/30 15:04:09.907 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/philhofer/fwd v1.1.1 [2024/05/30 15:04:10.107 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:04:10.216 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:04:10.298 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:04:10.311 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:04:10.315 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/aws/smithy-go v1.13.5 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/hashicorp/errwrap v1.0.0 [2024/05/30 15:04:10.796 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:04:10.796 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:04:10.807 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 table ddl_manager.finish_mark not exists for 39-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.12076.out cli unsafe resolve-lock --region=66 [2024/05/30 15:04:10.901 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/30 15:04:11.101 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:04:11.293 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:04:11 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16022ba8-d406-4043-a67f-e5a405ac7e22 {"id":"16022ba8-d406-4043-a67f-e5a405ac7e22","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ff456bf 16022ba8-d406-4043-a67f-e5a405ac7e22 /tidb/cdc/default/default/upstream/7374684911958876435 {"id":7374684911958876435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16022ba8-d406-4043-a67f-e5a405ac7e22 {"id":"16022ba8-d406-4043-a67f-e5a405ac7e22","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ff456bf 16022ba8-d406-4043-a67f-e5a405ac7e22 /tidb/cdc/default/default/upstream/7374684911958876435 {"id":7374684911958876435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16022ba8-d406-4043-a67f-e5a405ac7e22 {"id":"16022ba8-d406-4043-a67f-e5a405ac7e22","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ff456bf 16022ba8-d406-4043-a67f-e5a405ac7e22 /tidb/cdc/default/default/upstream/7374684911958876435 {"id":7374684911958876435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.7633.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c=ddl-only-block-related-table go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [2024/05/30 15:04:11.506 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:04:11.515 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:04:11.610 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:04:11.620 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [Pipeline] // cache [2024/05/30 15:04:11.909 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:04:11.915 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:04:11.994 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:04:12.012 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [Pipeline] } [Pipeline] // dir [Pipeline] } Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7374684911958876435,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:04:12.002093475+08:00","start_ts":450115050328555527,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115050328555527,"checkpoint_ts":450115050328555527,"checkpoint_time":"2024-05-30 15:04:11.705"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] // withCredentials [2024/05/30 15:04:12.314 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [2024/05/30 15:04:12.503 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs21ad2fa2_2e0b_41bf_9f0a_ff28a19abd0e"] [2024/05/30 15:04:12.595 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:04:12.600 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7bf2c76f_36a1_40f5_9030_90a1f26a5fac"] [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [2024/05/30 15:04:12.695 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [Pipeline] } table ddl_manager.finish_mark not exists for 40-th check, retry later [Pipeline] // node [Pipeline] } [2024/05/30 15:04:13.008 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:04:13.107 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [Pipeline] // podTemplate [Pipeline] } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.12107.out cli unsafe resolve-lock --region=66 --ts=450115049333981190 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... [Pipeline] // withEnv [Pipeline] } [2024/05/30 15:04:13.201 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/30 15:04:13.403 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs217ebcd3_81ba_42e4_b8ed_b5a8cfe96223"] [Pipeline] // stage [Pipeline] } + set +x [Thu May 30 15:04:13 CST 2024] <<<<<< START kafka consumer in ddl_only_block_related_table case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:04:13.715 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:04:13.908 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] go: downloading github.com/ardielle/ardielle-go v1.5.2 [2024/05/30 15:04:13.997 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:04:14.094 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:04:14.096 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:04:14.197 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/30 15:04:14.392 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLscca0c89e_2bb9_4b99_936b_cc26a89fdec9"] + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 96101 2719 --:--:-- --:--:-- --:--:-- 103k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" [2024/05/30 15:04:14.517 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:04:14.704 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] table ddl_manager.finish_mark not exists for 41-th check, retry later [2024/05/30 15:04:15.010 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:04:15.015 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:04:15.303 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/30 15:04:15.496 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:04:15.515 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:04:15.599 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:04:15.600 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:04:15.796 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:04:15.810 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:04:15.893 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later [2024/05/30 15:04:16.115 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:04:16.118 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:04:16.410 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] table ddl_manager.finish_mark not exists for 42-th check, retry later [2024/05/30 15:04:16.597 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:04:16.701 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:04:16.706 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:04:16.714 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:04:16.905 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:04:16.911 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/30 15:04:17.005 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/30 15:04:17.217 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:04:17.308 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:04:17.496 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsdc9647fc_7101_47e2_acc3_7aaac29ef830"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21411be00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:14199, start at 2024-05-30 15:04:15.395235841 +0800 CST m=+6.309021587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:06:15.405 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:04:15.402 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:54:15.402 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21411be00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:14199, start at 2024-05-30 15:04:15.395235841 +0800 CST m=+6.309021587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:06:15.405 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:04:15.402 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:54:15.402 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21411f9c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:14259, start at 2024-05-30 15:04:15.633071817 +0800 CST m=+6.432016650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:06:15.640 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:04:15.641 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:54:15.641 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/30 15:04:17.612 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 176 100 176 0 0 181k 0 --:--:-- --:--:-- --:--:-- 171k { "version": "v7.5.1-58-g8f92142d7", "git_hash": "8f92142d7874bfad2a8db6ca100604c383fca9df", "id": "5013ed65-e4f8-421f-bcfb-a4142816a2b5", "pid": 11270, "is_owner": true [Thu May 30 15:04:16 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 10 [Thu May 30 15:04:16 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 20 [Thu May 30 15:04:16 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: finish table test.finish not exists for 1-th check, retry later [2024/05/30 15:04:17.893 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:04:17.994 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:04:18.015 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later }wait process cdc.test exit for 1-th time... [2024/05/30 15:04:18.299 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:04:18.299 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table test.finish not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... [2024/05/30 15:04:18.614 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:04:18.722 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:04:18.907 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/30 15:04:18.995 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/30 15:04:19.095 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] table ddl_manager.finish_mark not exists for 43-th check, retry later wait process cdc.test exit for 3-th time... [2024/05/30 15:04:19.201 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:04:19.323 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:04:19.401 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:04:19.425 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/30 15:04:19.598 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:04:19 CST 2024] <<<<<< run test case cli success! >>>>>> [2024/05/30 15:04:19.821 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [Thu May 30 15:04:19 CST 2024] <<<<<< START cdc server in kafka_simple_claim_check_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.1548315485.out server --log-file /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/30 15:04:20.215 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:04:20.307 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later [2024/05/30 15:04:20.492 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/30 15:04:20.510 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:04:20.622 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:04:20.896 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:04:20.912 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:04:20.920 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table ddl_manager.finish_mark not exists for 44-th check, retry later table test.finish exists check diff successfully [2024/05/30 15:04:21.013 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/30 15:04:21.214 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] wait process cdc.test exit for 1-th time... [2024/05/30 15:04:21.600 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/30 15:04:21.605 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] wait process cdc.test exit for 2-th time... [2024/05/30 15:04:21.818 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/30 15:04:21.824 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:04:22.026 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] wait process cdc.test exit for 3-th time... table ddl_only_block_related_table.finish_mark exists [2024/05/30 15:04:22.422 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:04:22.433 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:04:22.501 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:04:22.524 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:04:22 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> wait process 7589 exit for 1-th time... [2024/05/30 15:04:22.711 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:04:22 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d28c10b-32aa-4c72-84e4-3c5f03b83308 {"id":"8d28c10b-32aa-4c72-84e4-3c5f03b83308","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc850186dc3 8d28c10b-32aa-4c72-84e4-3c5f03b83308 /tidb/cdc/default/default/upstream/7374684957938933254 {"id":7374684957938933254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d28c10b-32aa-4c72-84e4-3c5f03b83308 {"id":"8d28c10b-32aa-4c72-84e4-3c5f03b83308","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc850186dc3 8d28c10b-32aa-4c72-84e4-3c5f03b83308 /tidb/cdc/default/default/upstream/7374684957938933254 {"id":7374684957938933254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d28c10b-32aa-4c72-84e4-3c5f03b83308 {"id":"8d28c10b-32aa-4c72-84e4-3c5f03b83308","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc850186dc3 8d28c10b-32aa-4c72-84e4-3c5f03b83308 /tidb/cdc/default/default/upstream/7374684957938933254 {"id":7374684957938933254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.15535.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 45-th check, retry later [2024/05/30 15:04:23.014 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:04:23.015 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] wait process 7589 exit for 2-th time... wait process 7589 exit for 3-th time... table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (7589) - No such process wait process 7589 exit for 4-th time... process 7589 already exit [Thu May 30 15:04:24 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.77467748.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/30 15:04:23.221 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:04:23.297 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/30 15:04:23.414 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:04:23.708 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:04:23.709 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:04:23.798 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:04:23.814 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:04:23.997 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:04:24.207 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/30 15:04:24.295 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:04:24.519 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] + set +x + tso='450115053289996294 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115053289996294 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.15561.out cli changefeed create --start-ts=450115053289996294 '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-16401?protocol=simple&encoding-format=avro' -c kafka-simple-claim-check-avro --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/conf/changefeed.toml [2024/05/30 15:04:24.814 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Create changefeed successfully! ID: kafka-simple-claim-check-avro Info: {"upstream_id":7374684957938933254,"namespace":"default","id":"kafka-simple-claim-check-avro","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-16401?protocol=simple\u0026encoding-format=avro","create_time":"2024-05-30T15:04:25.028913963+08:00","start_ts":450115053289996294,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-avro-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115053289996294,"checkpoint_ts":450115053289996294,"checkpoint_time":"2024-05-30 15:04:23.002"} PASS [2024/05/30 15:04:24.913 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:04:25.317 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/30 15:04:25.328 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:04:25.415 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:04:25.518 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:04:25.626 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:04:25.828 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/30 15:04:25.921 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/30 15:04:26.212 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/30 15:04:26.223 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table test.finish_mark not exists for 5-th check, retry later [2024/05/30 15:04:26.423 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:04:26.526 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:04:26.618 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] + set +x [2024/05/30 15:04:26.712 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table ddl_manager.finish_mark not exists for 46-th check, retry later [2024/05/30 15:04:26.926 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/30 15:04:27.105 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/30 15:04:27.327 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/30 15:04:27.611 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:04:27 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7374684911958876435 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:12.002093475 +0800 CST StartTs:450115050328555527 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f8cd80 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115050394091534} {CheckpointTs:450115054064107528 MinTableBarrierTs:450115054326251529 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450115054326251530, checkpointTs: 450115054326251529, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450115054326251529, checkpointTs: 450115054326251529, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450115054326251529, checkpointTs: 450115054326251529, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 450115054326251529, checkpointTs: 450115054326251529, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a946d762-7e80-4baf-b1a0-36154d19d485 {"id":"a946d762-7e80-4baf-b1a0-36154d19d485","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ff45783 a946d762-7e80-4baf-b1a0-36154d19d485 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374684911958876435,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:12.002093475+08:00","start-ts":450115050328555527,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115050394091534} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450115054064107528,"min-table-barrier-ts":450115054326251529,"admin-job-type":0} /tidb/cdc/default/default/task/position/a946d762-7e80-4baf-b1a0-36154d19d485/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684911958876435 {"id":7374684911958876435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7374684911958876435 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:12.002093475 +0800 CST StartTs:450115050328555527 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f8cd80 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115050394091534} {CheckpointTs:450115054064107528 MinTableBarrierTs:450115054326251529 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450115054326251530, checkpointTs: 450115054326251529, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450115054326251529, checkpointTs: 450115054326251529, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450115054326251529, checkpointTs: 450115054326251529, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 450115054326251529, checkpointTs: 450115054326251529, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a946d762-7e80-4baf-b1a0-36154d19d485 {"id":"a946d762-7e80-4baf-b1a0-36154d19d485","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ff45783 a946d762-7e80-4baf-b1a0-36154d19d485 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374684911958876435,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:12.002093475+08:00","start-ts":450115050328555527,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115050394091534} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450115054064107528,"min-table-barrier-ts":450115054326251529,"admin-job-type":0} /tidb/cdc/default/default/task/position/a946d762-7e80-4baf-b1a0-36154d19d485/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684911958876435 {"id":7374684911958876435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7374684911958876435 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:12.002093475 +0800 CST StartTs:450115050328555527 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f8cd80 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115050394091534} {CheckpointTs:450115054064107528 MinTableBarrierTs:450115054326251529 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450115054326251530, checkpointTs: 450115054326251529, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450115054326251529, checkpointTs: 450115054326251529, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450115054326251529, checkpointTs: 450115054326251529, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 450115054326251529, checkpointTs: 450115054326251529, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a946d762-7e80-4baf-b1a0-36154d19d485 {"id":"a946d762-7e80-4baf-b1a0-36154d19d485","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ff45783 a946d762-7e80-4baf-b1a0-36154d19d485 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374684911958876435,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:12.002093475+08:00","start-ts":450115050328555527,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115050394091534} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450115054064107528,"min-table-barrier-ts":450115054326251529,"admin-job-type":0} /tidb/cdc/default/default/task/position/a946d762-7e80-4baf-b1a0-36154d19d485/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684911958876435 + grep -q 'etcd info' {"id":7374684911958876435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [2024/05/30 15:04:27.724 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/30 15:04:27.924 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] check_ts_not_forward ddl-only-block-related-table [2024/05/30 15:04:28.040 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/30 15:04:28.364 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/30 15:04:28.499 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table ddl_manager.finish_mark not exists for 47-th check, retry later table test.finish_mark not exists for 6-th check, retry later [2024/05/30 15:04:29.218 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v2m1.499358228s"] [2024/05/30 15:04:29.733 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 2m2.014013291s"] table mark.finish_mark_1 exists table test.finish_mark not exists for 7-th check, retry later table mark.finish_mark_2 exists table ddl_manager.finish_mark not exists for 48-th check, retry later table mark.finish_mark_3 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.15619.out cli changefeed pause -c kafka-simple-claim-check-avro run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 49-th check, retry later PASS table test.finish_mark not exists for 8-th check, retry later coverage: 1.9% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... check_ts_not_forward ddl-only-block-related-table =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=kafka + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run kafka + '[' kafka '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Thu May 30 15:04:34 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Thu May 30 15:04:34 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.15641.out cli changefeed update -c kafka-simple-claim-check-avro '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-16401?protocol=simple&encoding-format=avro&max-message-bytes=2048' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/conf/changefeed.toml --no-confirm table ddl_manager.finish_mark not exists for 50-th check, retry later Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-16401?protocol=simple&encoding-format=avro To:kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-16401?protocol=simple&encoding-format=avro&max-message-bytes=2048} {Type:update Path:[Config SyncPointInterval] From: To:0xc0034d0fb8} {Type:update Path:[Config SyncPointRetention] From: To:0xc0034d0fc8} {Type:update Path:[Config Consistent] From: To:0xc000d762a0} table test.finish_mark not exists for 9-th check, retry later Update changefeed config successfully! ID: kafka-simple-claim-check-avro Info: {"upstream_id":7374684957938933254,"namespace":"default","id":"kafka-simple-claim-check-avro","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-16401?protocol=simple\u0026encoding-format=avro\u0026max-message-bytes=2048","create_time":"2024-05-30T15:04:25.028913963+08:00","start_ts":450115053289996294,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-avro-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115055963865092,"checkpoint_ts":450115055714828293,"checkpoint_time":"2024-05-30 15:04:32.252"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 3-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.15669.out cli changefeed resume -c kafka-simple-claim-check-avro table ddl_manager.finish_mark not exists for 51-th check, retry later table test.finish_mark not exists for 10-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:04:37 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 52-th check, retry later table test.finish_mark not exists for 11-th check, retry later + set +x table test.finish_mark not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:04:40 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 12-th check, retry later table ddl_manager.finish_mark not exists for 53-th check, retry later table test.finish_mark not exists for 2-th check, retry later table mark.finish_mark_3 not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:04:43 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> table ddl_manager.finish_mark not exists for 54-th check, retry later table test.finish_mark exists check diff successfully table test.finish_mark not exists for 13-th check, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 7-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:04:45 CST 2024] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> table ddl_manager.finish_mark not exists for 55-th check, retry later [2024/05/30 15:04:43.946 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/30 15:04:43.948 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/30 15:04:43.949 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=45445e04-8969-4898-b0b2-509da30839a0] [2024/05/30 15:04:43.949 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/30 15:04:43.949 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/30 15:04:44.143 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8ffdd20c-58fb-4892-9cd5-d34fe4050a0a] [captureID=36140922-90e6-4ad8-8207-c14da8f49abe] [processorDetail="{\"table_ids\":[106]}"] [2024/05/30 15:04:44.342 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8ffdd20c-58fb-4892-9cd5-d34fe4050a0a] [captureID=45445e04-8969-4898-b0b2-509da30839a0] [processorDetail="{\"table_ids\":[104]}"] [2024/05/30 15:04:44.542 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8ffdd20c-58fb-4892-9cd5-d34fe4050a0a] [captureID=6f6a055e-cb62-4382-95d5-9df8e1b9f803] [processorDetail="{\"table_ids\":[]}"] [2024/05/30 15:04:44.542 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"36140922-90e6-4ad8-8207-c14da8f49abe\":[{\"ID\":106,\"Changefeed\":\"8ffdd20c-58fb-4892-9cd5-d34fe4050a0a\"}],\"45445e04-8969-4898-b0b2-509da30839a0\":[{\"ID\":104,\"Changefeed\":\"8ffdd20c-58fb-4892-9cd5-d34fe4050a0a\"}],\"6f6a055e-cb62-4382-95d5-9df8e1b9f803\":[]}"] [2024/05/30 15:04:44.542 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=8ffdd20c-58fb-4892-9cd5-d34fe4050a0a&target-cp-id=6f6a055e-cb62-4382-95d5-9df8e1b9f803&table-id=106"] [2024/05/30 15:04:44.592 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=106] [2024/05/30 15:04:44.592 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=36140922-90e6-4ad8-8207-c14da8f49abe] [targetCapture=6f6a055e-cb62-4382-95d5-9df8e1b9f803] table move_table.check1 exists check diff successfully table test.finish_mark not exists for 14-th check, retry later table mark.finish_mark_3 not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [2024/05/30 15:04:42.700 +08:00] [WARN] [diff.go:556] ["checksum is not equal"] [table=`test`.`many_cols`] [where="((TRUE) AND TRUE)"] [sourceChecksum=3564127999] [targetChecksum=3242799125] [getSourceChecksumCost=1.447255ms] [getTargetChecksumCost=1.120808ms] [2024/05/30 15:04:42.701 +08:00] [WARN] [diff.go:922] ["find different row"] [column=val] [row1="{ id: 1, val: 2085, }"] [row2="{ id: 1, val: 2067, }"] [2024/05/30 15:04:42.791 +08:00] [WARN] [diff.go:765] ["rows is not equal"] [table=`test`.`many_cols`] [where="((TRUE) AND TRUE)"] [cost=91.612659ms] [2024/05/30 15:04:42.793 +08:00] [WARN] [diff.go:383] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table test.finish_mark not exists for 15-th check, retry later [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table ddl_manager.finish_mark not exists for 56-th check, retry later table mark.finish_mark_3 not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21430370000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:13007, start at 2024-05-30 15:04:46.595414334 +0800 CST m=+6.069685787 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:06:46.603 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:04:46.605 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:54:46.605 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21430370000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:13007, start at 2024-05-30 15:04:46.595414334 +0800 CST m=+6.069685787 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:06:46.603 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:04:46.605 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:54:46.605 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214303440011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:13084, start at 2024-05-30 15:04:46.591807538 +0800 CST m=+5.918896542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:06:46.600 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:04:46.595 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:54:46.595 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark exists check diff successfully table ddl_manager.finish_mark not exists for 57-th check, retry later table mark.finish_mark_3 not exists for 10-th check, retry later wait process cdc.test exit for 1-th time... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.14352.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... [2024/05/30 15:04:50.530 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/30 15:04:50.532 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/30 15:04:50.532 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=45445e04-8969-4898-b0b2-509da30839a0] [2024/05/30 15:04:50.533 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/30 15:04:50.533 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/30 15:04:50.693 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8ffdd20c-58fb-4892-9cd5-d34fe4050a0a] [captureID=36140922-90e6-4ad8-8207-c14da8f49abe] [processorDetail="{\"table_ids\":[]}"] [2024/05/30 15:04:50.892 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8ffdd20c-58fb-4892-9cd5-d34fe4050a0a] [captureID=45445e04-8969-4898-b0b2-509da30839a0] [processorDetail="{\"table_ids\":[108]}"] [2024/05/30 15:04:51.092 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8ffdd20c-58fb-4892-9cd5-d34fe4050a0a] [captureID=6f6a055e-cb62-4382-95d5-9df8e1b9f803] [processorDetail="{\"table_ids\":[106]}"] [2024/05/30 15:04:51.092 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"36140922-90e6-4ad8-8207-c14da8f49abe\":[],\"45445e04-8969-4898-b0b2-509da30839a0\":[{\"ID\":108,\"Changefeed\":\"8ffdd20c-58fb-4892-9cd5-d34fe4050a0a\"}],\"6f6a055e-cb62-4382-95d5-9df8e1b9f803\":[{\"ID\":106,\"Changefeed\":\"8ffdd20c-58fb-4892-9cd5-d34fe4050a0a\"}]}"] [2024/05/30 15:04:51.092 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=8ffdd20c-58fb-4892-9cd5-d34fe4050a0a&target-cp-id=6f6a055e-cb62-4382-95d5-9df8e1b9f803&table-id=108"] [2024/05/30 15:04:51.142 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/05/30 15:04:51.142 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=45445e04-8969-4898-b0b2-509da30839a0] [targetCapture=6f6a055e-cb62-4382-95d5-9df8e1b9f803] check diff successfully table move_table.check2 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... table ddl_manager.finish_mark not exists for 58-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:04:52 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> + set +x + tso='450115060709457921 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115060709457921 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:04:52 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1437714379.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table move_table.check2 exists check diff successfully table mark.finish_mark_3 not exists for 11-th check, retry later wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 59-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table mark.finish_mark_3 not exists for 12-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:04:55 CST 2024] <<<<<< run test case move_table success! >>>>>> table ddl_manager.finish_mark not exists for 60-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:04:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20513659-65cf-4c52-b1c8-48397e69616a {"id":"20513659-65cf-4c52-b1c8-48397e69616a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc850915fc2 20513659-65cf-4c52-b1c8-48397e69616a /tidb/cdc/default/default/upstream/7374685096979103150 {"id":7374685096979103150,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20513659-65cf-4c52-b1c8-48397e69616a {"id":"20513659-65cf-4c52-b1c8-48397e69616a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc850915fc2 20513659-65cf-4c52-b1c8-48397e69616a /tidb/cdc/default/default/upstream/7374685096979103150 {"id":7374685096979103150,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20513659-65cf-4c52-b1c8-48397e69616a {"id":"20513659-65cf-4c52-b1c8-48397e69616a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc850915fc2 20513659-65cf-4c52-b1c8-48397e69616a /tidb/cdc/default/default/upstream/7374685096979103150 {"id":7374685096979103150,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.14430.out cli changefeed create --start-ts=450115060709457921 '--sink-uri=kafka://127.0.0.1:9092/ticdc-savepoint-test-6678?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: a010a474-a992-4005-966a-1c52f253b5d2 Info: {"upstream_id":7374685096979103150,"namespace":"default","id":"a010a474-a992-4005-966a-1c52f253b5d2","sink_uri":"kafka://127.0.0.1:9092/ticdc-savepoint-test-6678?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:04:56.319160425+08:00","start_ts":450115060709457921,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115060709457921,"checkpoint_ts":450115060709457921,"checkpoint_time":"2024-05-30 15:04:51.305"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh: line 1: 15693 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 13-th check, retry later + set +x [Thu May 30 15:04:57 CST 2024] <<<<<< START kafka consumer in savepoint case >>>>>> table ddl_manager.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table savepoint.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... table mark.finish_mark_3 not exists for 14-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/canal_json_adapter_compatibility Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... wait process cdc.test exit for 4-th time... wait process cdc.test exit for 5-th time... wait process cdc.test exit for 6-th time... table savepoint.finish_mark exists check diff successfully wait process cdc.test exit for 7-th time... table mark.finish_mark_3 not exists for 15-th check, retry later wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 8-th time... process cdc.test already exit [Thu May 30 15:05:02 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:05:03 CST 2024] <<<<<< run test case savepoint success! >>>>>> table mark.finish_mark_3 not exists for 16-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 17-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... table mark.finish_mark_3 not exists for 18-th check, retry later table test1.finishmark exists [2024/05/30 15:05:02.280 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/05/30 15:05:02.285 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/05/30 15:05:02.290 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/05/30 15:05:02.295 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/05/30 15:05:02.298 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/05/30 15:05:02.298 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=21.92968ms] [2024/05/30 15:05:02.302 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/05/30 15:05:02.304 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/05/30 15:05:02.306 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/05/30 15:05:02.311 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/05/30 15:05:02.313 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/05/30 15:05:02.314 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=15.331588ms] [2024/05/30 15:05:02.314 +08:00] [INFO] [main.go:95] ["compare checksum passed"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:05:04 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 19-th check, retry later table big_txn.finish_mark_1 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:05:00 CST 2024] <<<<<< run test case big_txn success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 20-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214472dc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:16696, start at 2024-05-30 15:05:10.115512194 +0800 CST m=+5.747337082 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:10.123 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:10.121 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:10.121 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214472dc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:16696, start at 2024-05-30 15:05:10.115512194 +0800 CST m=+5.747337082 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:10.123 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:10.121 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:10.121 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214474680013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:16772, start at 2024-05-30 15:05:10.198351083 +0800 CST m=+5.719752117 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:10.206 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:10.170 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:10.170 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 21-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.cli.18013.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=kafka + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450115066939572229 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115066939572229 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:05:16 CST 2024] <<<<<< START cdc server in canal_json_adapter_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.1804518047.out server --log-file /tmp/tidb_cdc_test/canal_json_adapter_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_adapter_compatibility/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh: line 1: 15081 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 (wd: /tmp/tidb_cdc_test/kafka_column_selector) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: kafka... <<================= Starting schema registry... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 table mark.finish_mark_3 not exists for 22-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 table mark.finish_mark_3 not exists for 23-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2144dfd80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l, pid:19832, start at 2024-05-30 15:05:17.07054742 +0800 CST m=+5.761755111 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:17.078 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:17.046 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:17.046 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2144dfd80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l, pid:19832, start at 2024-05-30 15:05:17.07054742 +0800 CST m=+5.761755111 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:17.078 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:17.046 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:17.046 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2144f420000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l, pid:19906, start at 2024-05-30 15:05:18.35607123 +0800 CST m=+6.930334905 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:18.361 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:18.344 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:18.344 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8088 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8088 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:05:19 GMT < Content-Type: application/vnd.schemaregistry.v1+json < Vary: Accept-Encoding, User-Agent < Content-Length: 2 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 49 100 24 100 25 345 359 --:--:-- --:--:-- --:--:-- 362 {"compatibility":"NONE"}The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:05:19 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6556115a-77ab-4c76-97fd-09a4713e2c4f {"id":"6556115a-77ab-4c76-97fd-09a4713e2c4f","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc850f443c1 6556115a-77ab-4c76-97fd-09a4713e2c4f /tidb/cdc/default/default/upstream/7374685199791142666 {"id":7374685199791142666,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6556115a-77ab-4c76-97fd-09a4713e2c4f {"id":"6556115a-77ab-4c76-97fd-09a4713e2c4f","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc850f443c1 6556115a-77ab-4c76-97fd-09a4713e2c4f /tidb/cdc/default/default/upstream/7374685199791142666 {"id":7374685199791142666,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6556115a-77ab-4c76-97fd-09a4713e2c4f {"id":"6556115a-77ab-4c76-97fd-09a4713e2c4f","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc850f443c1 6556115a-77ab-4c76-97fd-09a4713e2c4f /tidb/cdc/default/default/upstream/7374685199791142666 {"id":7374685199791142666,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.cli.18088.out cli changefeed create --start-ts=450115066939572229 '--sink-uri=kafka://127.0.0.1:9092/test?protocol=canal-json&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 1b9f714a-1a62-44f2-a67e-d46f423d67ea Info: {"upstream_id":7374685199791142666,"namespace":"default","id":"1b9f714a-1a62-44f2-a67e-d46f423d67ea","sink_uri":"kafka://127.0.0.1:9092/test?protocol=canal-json\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:05:20.111573182+08:00","start_ts":450115066939572229,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115066939572229,"checkpoint_ts":450115066939572229,"checkpoint_time":"2024-05-30 15:05:15.071"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: kafka... <<================= table mark.finish_mark_3 not exists for 24-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 25-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214530a80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:27702, start at 2024-05-30 15:05:22.237279287 +0800 CST m=+5.856303127 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:22.244 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:22.218 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:22.218 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.21157.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector_avro Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... The 1 times to try to start tidb cluster... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214530a80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:27702, start at 2024-05-30 15:05:22.237279287 +0800 CST m=+5.856303127 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:22.244 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:22.218 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:22.218 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214533100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:27774, start at 2024-05-30 15:05:22.41596761 +0800 CST m=+5.926437214 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:22.423 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:22.420 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:22.420 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='450115069122183174 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115069122183174 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:05:24 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2119021192.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 26-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Thu May 30 15:05:26 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.2907029072.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 27-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:05:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ece1848a-3a0f-4fc2-85cf-6f78f6cc4e12 {"id":"ece1848a-3a0f-4fc2-85cf-6f78f6cc4e12","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8510f55e6 ece1848a-3a0f-4fc2-85cf-6f78f6cc4e12 /tidb/cdc/default/default/upstream/7374685224559143063 {"id":7374685224559143063,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ece1848a-3a0f-4fc2-85cf-6f78f6cc4e12 {"id":"ece1848a-3a0f-4fc2-85cf-6f78f6cc4e12","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8510f55e6 ece1848a-3a0f-4fc2-85cf-6f78f6cc4e12 /tidb/cdc/default/default/upstream/7374685224559143063 {"id":7374685224559143063,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ece1848a-3a0f-4fc2-85cf-6f78f6cc4e12 {"id":"ece1848a-3a0f-4fc2-85cf-6f78f6cc4e12","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8510f55e6 ece1848a-3a0f-4fc2-85cf-6f78f6cc4e12 /tidb/cdc/default/default/upstream/7374685224559143063 {"id":7374685224559143063,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:05:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfec45e9-500e-4dff-8835-29d445effe2d {"id":"dfec45e9-500e-4dff-8835-29d445effe2d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8511f39c6 dfec45e9-500e-4dff-8835-29d445effe2d /tidb/cdc/default/default/upstream/7374685246598277172 {"id":7374685246598277172,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfec45e9-500e-4dff-8835-29d445effe2d {"id":"dfec45e9-500e-4dff-8835-29d445effe2d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8511f39c6 dfec45e9-500e-4dff-8835-29d445effe2d /tidb/cdc/default/default/upstream/7374685246598277172 {"id":7374685246598277172,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfec45e9-500e-4dff-8835-29d445effe2d {"id":"dfec45e9-500e-4dff-8835-29d445effe2d","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8511f39c6 dfec45e9-500e-4dff-8835-29d445effe2d /tidb/cdc/default/default/upstream/7374685246598277172 {"id":7374685246598277172,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2145a39c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:15403, start at 2024-05-30 15:05:29.598273829 +0800 CST m=+5.824976230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:29.606 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:29.575 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:29.575 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 127254a9-4600-4973-b69a-50cc7f75f3dd Info: {"upstream_id":7374685246598277172,"namespace":"default","id":"127254a9-4600-4973-b69a-50cc7f75f3dd","sink_uri":"kafka://127.0.0.1:9092/ticdc-autorandom-test-23584?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:05:30.141818208+08:00","start_ts":450115070858100737,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115070858100737,"checkpoint_ts":450115070858100737,"checkpoint_time":"2024-05-30 15:05:30.019"} [Thu May 30 15:05:30 CST 2024] <<<<<< START kafka consumer in autorandom case >>>>>> table autorandom_test.table_a not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 28-th check, retry later [Thu May 30 15:05:30 CST 2024] <<<<<< START kafka consumer in capture_session_done_during_task case >>>>>> lease 22318fc8510f55e6 revoked ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.binary_columns not exists for 1-th check, retry later table capture_session_done_during_task.t exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2145ba080004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf, pid:3901, start at 2024-05-30 15:05:31.014080086 +0800 CST m=+5.588292386 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:31.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:31.010 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:31.010 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2145a39c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:15403, start at 2024-05-30 15:05:29.598273829 +0800 CST m=+5.824976230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:29.606 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:29.575 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:29.575 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2145a5f00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:15468, start at 2024-05-30 15:05:29.763712879 +0800 CST m=+5.885151311 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:29.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:29.774 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:29.774 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_3 not exists for 29-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table autorandom_test.table_a not exists for 2-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.binary_columns not exists for 2-th check, retry later Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2145ba080004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf, pid:3901, start at 2024-05-30 15:05:31.014080086 +0800 CST m=+5.588292386 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:31.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:31.010 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:31.010 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2145bb7c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf, pid:3986, start at 2024-05-30 15:05:31.115504545 +0800 CST m=+5.584689834 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:31.125 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:31.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:31.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/30 15:05:31.117 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 11.725002556s"] check diff failed 2-th time, retry later table mark.finish_mark_3 not exists for 30-th check, retry later + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.16739.out cli tso query --pd=http://127.0.0.1:2379 table autorandom_test.table_a exists check diff successfully wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2145eef8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:27406, start at 2024-05-30 15:05:34.409068237 +0800 CST m=+6.001069404 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:34.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:34.398 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:34.398 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.binary_columns not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... wait process cdc.test exit for 2-th time... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later + set +x + tso='450115072039583745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115072039583745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450115072039583745 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Thu May 30 15:05:35 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1677316775.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 31-th check, retry later wait process cdc.test exit for 3-th time... [Thu May 30 15:05:35 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.52095211.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:05:36 CST 2024] <<<<<< run test case autorandom success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2145eef8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:27406, start at 2024-05-30 15:05:34.409068237 +0800 CST m=+6.001069404 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:34.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:34.398 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:34.398 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2145efac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-40hs7, pid:27493, start at 2024-05-30 15:05:34.487671487 +0800 CST m=+5.908815568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:34.495 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:34.493 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:34.493 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table test.binary_columns not exists for 4-th check, retry later table mark.finish_mark_3 not exists for 32-th check, retry later check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [Thu May 30 15:05:38 CST 2024] <<<<<< START cdc server in kafka_column_selector_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.2874028742.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:05:38 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05b9e7ab-df83-4fc0-a8b9-37531ac648e0 {"id":"05b9e7ab-df83-4fc0-a8b9-37531ac648e0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85140cabd 05b9e7ab-df83-4fc0-a8b9-37531ac648e0 /tidb/cdc/default/default/upstream/7374685287633591945 {"id":7374685287633591945,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05b9e7ab-df83-4fc0-a8b9-37531ac648e0 {"id":"05b9e7ab-df83-4fc0-a8b9-37531ac648e0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85140cabd 05b9e7ab-df83-4fc0-a8b9-37531ac648e0 /tidb/cdc/default/default/upstream/7374685287633591945 {"id":7374685287633591945,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05b9e7ab-df83-4fc0-a8b9-37531ac648e0 {"id":"05b9e7ab-df83-4fc0-a8b9-37531ac648e0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85140cabd 05b9e7ab-df83-4fc0-a8b9-37531ac648e0 /tidb/cdc/default/default/upstream/7374685287633591945 {"id":7374685287633591945,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Thu May 30 15:05:39 CST 2024] <<<<<< START kafka consumer in changefeed_finish case >>>>>> table test.binary_columns not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:05:39 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28b9d30a-56df-4df1-9b39-8e321ab0c2fb {"id":"28b9d30a-56df-4df1-9b39-8e321ab0c2fb","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8513a34c5 28b9d30a-56df-4df1-9b39-8e321ab0c2fb /tidb/cdc/default/default/upstream/7374685272019082750 {"id":7374685272019082750,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28b9d30a-56df-4df1-9b39-8e321ab0c2fb {"id":"28b9d30a-56df-4df1-9b39-8e321ab0c2fb","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8513a34c5 28b9d30a-56df-4df1-9b39-8e321ab0c2fb /tidb/cdc/default/default/upstream/7374685272019082750 {"id":7374685272019082750,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28b9d30a-56df-4df1-9b39-8e321ab0c2fb {"id":"28b9d30a-56df-4df1-9b39-8e321ab0c2fb","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8513a34c5 28b9d30a-56df-4df1-9b39-8e321ab0c2fb /tidb/cdc/default/default/upstream/7374685272019082750 {"id":7374685272019082750,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450115072039583745 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.16827.out cli changefeed create --start-ts=450115072039583745 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374685272019082750,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-30T15:05:39.500980183+08:00","start_ts":450115072039583745,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115072039583745,"checkpoint_ts":450115072039583745,"checkpoint_time":"2024-05-30 15:05:34.526"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... table mark.finish_mark_3 not exists for 33-th check, retry later + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 3185 0 --:--:-- --:--:-- --:--:-- 3202 wait process cdc.test exit for 2-th time... + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-30 15:05:40.875","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-30 15:05:40.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:05:40.875","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:05:40.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:05:40.875","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:05:40.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-30 15:05:40.875' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:05:40.875","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:05:40.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:05:40.875","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:05:40.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-30 15:05:41' + echo 'sink_checkpoint_ts is 2024-05-30' 15:05:40.875 sink_checkpoint_ts is 2024-05-30 15:05:40.875 ++ date -d '2024-05-30 15:05:40.875' +%s + checkpoint_timestamp=1717052740 ++ date -d '2024-05-30 15:05:41' +%s + current_timestamp=1717052741 + '[' 1 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:05:41 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> check diff failed 2-th time, retry later table test.binary_columns not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass check, checkpoint tso not forward after 10s run task successfully wait process 7751 exit for 1-th time... wait process 7751 exit for 2-th time... wait process 7751 exit for 3-th time... wait process 7751 exit for 4-th time... wait process 7751 exit for 5-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (7751) - No such process wait process 7751 exit for 6-th time... process 7751 already exit [Thu May 30 15:05:36 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.83238325.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:05:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7374684911958876435 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:12.002093475 +0800 CST StartTs:450115050328555527 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00343b830 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115050394091534} {CheckpointTs:450115054470430731 MinTableBarrierTs:450115073226833924 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18800c42-fd7b-4f87-a3f4-72b3222db4ee {"id":"18800c42-fd7b-4f87-a3f4-72b3222db4ee","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ff4594d 18800c42-fd7b-4f87-a3f4-72b3222db4ee /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374684911958876435,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:12.002093475+08:00","start-ts":450115050328555527,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115050394091534} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450115054470430731,"min-table-barrier-ts":450115073226833924,"admin-job-type":0} /tidb/cdc/default/default/task/position/18800c42-fd7b-4f87-a3f4-72b3222db4ee/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684911958876435 {"id":7374684911958876435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7374684911958876435 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:12.002093475 +0800 CST StartTs:450115050328555527 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00343b830 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115050394091534} {CheckpointTs:450115054470430731 MinTableBarrierTs:450115073226833924 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18800c42-fd7b-4f87-a3f4-72b3222db4ee {"id":"18800c42-fd7b-4f87-a3f4-72b3222db4ee","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ff4594d 18800c42-fd7b-4f87-a3f4-72b3222db4ee /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374684911958876435,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:12.002093475+08:00","start-ts":450115050328555527,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115050394091534} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450115054470430731,"min-table-barrier-ts":450115073226833924,"admin-job-type":0} /tidb/cdc/default/default/task/position/18800c42-fd7b-4f87-a3f4-72b3222db4ee/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684911958876435 {"id":7374684911958876435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7374684911958876435 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-30 15:04:12.002093475 +0800 CST StartTs:450115050328555527 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00343b830 State:normal Error: Warning: CreatorVersion:v7.5.1-58-g8f92142d7 Epoch:450115050394091534} {CheckpointTs:450115054470430731 MinTableBarrierTs:450115073226833924 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450115054470430731, checkpointTs: 450115054470430731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18800c42-fd7b-4f87-a3f4-72b3222db4ee {"id":"18800c42-fd7b-4f87-a3f4-72b3222db4ee","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc84ff4594d 18800c42-fd7b-4f87-a3f4-72b3222db4ee /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7374684911958876435,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-6342?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-30T15:04:12.002093475+08:00","start-ts":450115050328555527,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-58-g8f92142d7","epoch":450115050394091534} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450115054470430731,"min-table-barrier-ts":450115073226833924,"admin-job-type":0} /tidb/cdc/default/default/task/position/18800c42-fd7b-4f87-a3f4-72b3222db4ee/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7374684911958876435 {"id":7374684911958876435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff failed 1-th time, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:05:42 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1841c0a4-7291-4465-a774-741c5f472112 {"id":"1841c0a4-7291-4465-a774-741c5f472112","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851505ebf 1841c0a4-7291-4465-a774-741c5f472112 /tidb/cdc/default/default/upstream/7374685302055356942 {"id":7374685302055356942,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1841c0a4-7291-4465-a774-741c5f472112 {"id":"1841c0a4-7291-4465-a774-741c5f472112","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851505ebf 1841c0a4-7291-4465-a774-741c5f472112 /tidb/cdc/default/default/upstream/7374685302055356942 {"id":7374685302055356942,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1841c0a4-7291-4465-a774-741c5f472112 {"id":"1841c0a4-7291-4465-a774-741c5f472112","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851505ebf 1841c0a4-7291-4465-a774-741c5f472112 /tidb/cdc/default/default/upstream/7374685302055356942 {"id":7374685302055356942,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.cli.28783.out cli changefeed create --start-ts=450115073198784516 '--sink-uri=kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro&enable-tidb-extension=true&avro-enable-watermark=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml --schema-registry=http://127.0.0.1:8088 Create changefeed successfully! ID: test Info: {"upstream_id":7374685302055356942,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro\u0026enable-tidb-extension=true\u0026avro-enable-watermark=true","create_time":"2024-05-30T15:05:42.547375285+08:00","start_ts":450115073198784516,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"avro","schema_registry":"http://127.0.0.1:8088","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":[{"matcher":["test.*"],"columns":["*","!b"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115073198784516,"checkpoint_ts":450115073198784516,"checkpoint_time":"2024-05-30 15:05:38.948"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 34-th check, retry later table test.t1 exists + sleep 5 table test.binary_columns exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21465a140010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr, pid:6459, start at 2024-05-30 15:05:41.270154611 +0800 CST m=+5.830873520 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:41.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:41.253 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:41.253 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21465a140010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr, pid:6459, start at 2024-05-30 15:05:41.270154611 +0800 CST m=+5.830873520 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:41.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:41.253 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:41.253 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21465abc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-qx7wr, pid:6521, start at 2024-05-30 15:05:41.326931152 +0800 CST m=+5.793102671 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:07:41.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:05:41.295 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:55:41.295 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully check_ts_forward ddl-only-block-related-table + set +x [Thu May 30 15:05:43 CST 2024] <<<<<< START kafka consumer in kafka_column_selector_avro case >>>>>> consumer replica config found: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml schema registry uri found: http://127.0.0.1:8088 check diff successfully Starting build checksum checker... table test.finishmark not exists for 1-th check, retry later table mark.finish_mark_3 exists table mark.finish_mark not exists for 1-th check, retry later check diff successfully + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.7805.out cli tso query --pd=http://127.0.0.1:2379 changefeed is working normally rts: 450115074537553900->450115074799697900 checkpoint: 450115074537553900->450115074799697900 run task successfully table mark.finish_mark exists wait process cdc.test exit for 1-th time... table test.finishmark not exists for 2-th check, retry later check diff successfully wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... + set +x + tso='450115075087532034 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115075087532034 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:05:47 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.78367838.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:05:48 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> wait process cdc.test exit for 3-th time... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3579 0 --:--:-- --:--:-- --:--:-- 3626 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-30 15:05:47.875","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-30 15:05:41.225","now_ts":"2024-05-30 15:05:48.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:05:47.875","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:05:41.225","now_ts":"2024-05-30' '15:05:48.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:05:47.875","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:05:41.225","now_ts":"2024-05-30' '15:05:48.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table test.finishmark not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:05:49 CST 2024] <<<<<< run test case default_value success! >>>>>> table test.finishmark exists [2024/05/30 15:05:50.546 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/05/30 15:05:50.548 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/05/30 15:05:50.548 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=9.683229ms] [2024/05/30 15:05:50.555 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/05/30 15:05:50.557 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/05/30 15:05:50.557 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=9.12297ms] [2024/05/30 15:05:50.557 +08:00] [INFO] [main.go:95] ["compare checksum passed"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:05:50 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3a3801f-274a-462b-96e0-d6775dfc75b4 {"id":"b3a3801f-274a-462b-96e0-d6775dfc75b4","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8516ba4c3 b3a3801f-274a-462b-96e0-d6775dfc75b4 /tidb/cdc/default/default/upstream/7374685325978727445 {"id":7374685325978727445,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3a3801f-274a-462b-96e0-d6775dfc75b4 {"id":"b3a3801f-274a-462b-96e0-d6775dfc75b4","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8516ba4c3 b3a3801f-274a-462b-96e0-d6775dfc75b4 /tidb/cdc/default/default/upstream/7374685325978727445 {"id":7374685325978727445,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3a3801f-274a-462b-96e0-d6775dfc75b4 {"id":"b3a3801f-274a-462b-96e0-d6775dfc75b4","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8516ba4c3 b3a3801f-274a-462b-96e0-d6775dfc75b4 /tidb/cdc/default/default/upstream/7374685325978727445 {"id":7374685325978727445,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.7879.out cli changefeed create --start-ts=450115075087532034 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-source-test-21026?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: ae3a03b5-6e68-496b-a34f-2f0ebae6e4e0 Info: {"upstream_id":7374685325978727445,"namespace":"default","id":"ae3a03b5-6e68-496b-a34f-2f0ebae6e4e0","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-source-test-21026?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:05:51.129397833+08:00","start_ts":450115075087532034,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115075087532034,"checkpoint_ts":450115075087532034,"checkpoint_time":"2024-05-30 15:05:46.153"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:05:52 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> + set +x [Thu May 30 15:05:52 CST 2024] <<<<<< START kafka consumer in multi_source case >>>>>> go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240527072219-a97e6464c01d go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240527072219-a97e6464c01d go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/text v0.15.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/kvproto v0.0.0-20240522024016-df42997c2c57 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/prometheus/client_golang v1.19.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240510070134-e7f530c64790 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 go: downloading github.com/pingcap/tipb v0.0.0-20240507090649-2bf6bb0cb996 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/tikv/pd/client v0.0.0-20240210135946-3488a653ddd9 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading golang.org/x/sys v0.20.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/stretchr/testify v1.9.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading golang.org/x/tools v0.21.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.6 go: downloading google.golang.org/api v0.170.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading google.golang.org/protobuf v1.34.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading golang.org/x/net v0.25.0 start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/procfs v0.15.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/sasha-s/go-deadlock v0.3.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/DataDog/zstd v1.5.5 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:05:59.475 +08:00] [WARN] [diff.go:179] ["table struct is not equal"] [reason="column name not equal, one is sEaxT another is c"] table test.binary_columns exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21478ea80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:30048, start at 2024-05-30 15:06:01.046963687 +0800 CST m=+5.751197139 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:01.065 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:01.057 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:01.057 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21478ea80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:30048, start at 2024-05-30 15:06:01.046963687 +0800 CST m=+5.751197139 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:01.065 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:01.057 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:01.057 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21478f700013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cpr5b, pid:30128, start at 2024-05-30 15:06:01.077564 +0800 CST m=+5.687818709 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:01.084 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:01.052 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:01.052 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:06:03 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> check diff failed 4-th time, retry later Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.31469.out cli tso query --pd=http://127.0.0.1:2379 check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 2-th time... + set +x + tso='450115080290828289 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115080290828289 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:06:07 CST 2024] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.3149231494.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:06:07 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:06:08 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2147f2480001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l, pid:22414, start at 2024-05-30 15:06:07.397045103 +0800 CST m=+6.028234503 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:07.411 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:07.378 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:07.378 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2147f2480001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l, pid:22414, start at 2024-05-30 15:06:07.397045103 +0800 CST m=+6.028234503 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:07.411 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:07.378 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:07.378 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2147f3000011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-lww8l, pid:22484, start at 2024-05-30 15:06:07.447389643 +0800 CST m=+5.940182735 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:07.454 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:07.424 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:07.424 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/30 15:06:10.155 +08:00] [WARN] [diff.go:556] ["checksum is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [sourceChecksum=0] [targetChecksum=2856244851] [getSourceChecksumCost=1.607087ms] [getTargetChecksumCost=1.194145ms] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:06:10 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6536df36-0696-4994-9a27-c4e9b7d7a147 {"id":"6536df36-0696-4994-9a27-c4e9b7d7a147","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851bd5dc4 6536df36-0696-4994-9a27-c4e9b7d7a147 /tidb/cdc/default/default/upstream/7374685418322862774 {"id":7374685418322862774,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6536df36-0696-4994-9a27-c4e9b7d7a147 {"id":"6536df36-0696-4994-9a27-c4e9b7d7a147","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851bd5dc4 6536df36-0696-4994-9a27-c4e9b7d7a147 /tidb/cdc/default/default/upstream/7374685418322862774 {"id":7374685418322862774,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6536df36-0696-4994-9a27-c4e9b7d7a147 {"id":"6536df36-0696-4994-9a27-c4e9b7d7a147","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851bd5dc4 6536df36-0696-4994-9a27-c4e9b7d7a147 /tidb/cdc/default/default/upstream/7374685418322862774 {"id":7374685418322862774,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.31544.out cli changefeed create --start-ts=450115080290828289 '--sink-uri=kafka://127.0.0.1:9092/ticdc-generate-column-test-4692?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 0f8d230e-3ef2-4213-9a7d-be7952ea00df Info: {"upstream_id":7374685418322862774,"namespace":"default","id":"0f8d230e-3ef2-4213-9a7d-be7952ea00df","sink_uri":"kafka://127.0.0.1:9092/ticdc-generate-column-test-4692?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:06:11.042788432+08:00","start_ts":450115080290828289,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115080290828289,"checkpoint_ts":450115080290828289,"checkpoint_time":"2024-05-30 15:06:06.002"} PASS [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.23774.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x [Thu May 30 15:06:12 CST 2024] <<<<<< START kafka consumer in generate_column case >>>>>> table generate_column.t not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table generate_column.t not exists for 2-th check, retry later + set +x + tso='450115081909829633 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115081909829633 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:06:14 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2380123803.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table generate_column.t exists table generate_column.t1 exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214867a40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:9536, start at 2024-05-30 15:06:14.911549111 +0800 CST m=+6.116054388 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:14.919 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:14.889 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:14.889 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214867a40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:9536, start at 2024-05-30 15:06:14.911549111 +0800 CST m=+6.116054388 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:14.919 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:14.889 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:14.889 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21486968000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-5v4j4, pid:9607, start at 2024-05-30 15:06:15.014439131 +0800 CST m=+6.110720760 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:15.025 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:15.002 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:15.002 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:06:18 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1b163bb-8db2-419b-88bf-1b4876aebf55 {"id":"c1b163bb-8db2-419b-88bf-1b4876aebf55","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851d130c8 c1b163bb-8db2-419b-88bf-1b4876aebf55 /tidb/cdc/default/default/upstream/7374685442892162163 {"id":7374685442892162163,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1b163bb-8db2-419b-88bf-1b4876aebf55 {"id":"c1b163bb-8db2-419b-88bf-1b4876aebf55","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851d130c8 c1b163bb-8db2-419b-88bf-1b4876aebf55 /tidb/cdc/default/default/upstream/7374685442892162163 {"id":7374685442892162163,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1b163bb-8db2-419b-88bf-1b4876aebf55 {"id":"c1b163bb-8db2-419b-88bf-1b4876aebf55","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851d130c8 c1b163bb-8db2-419b-88bf-1b4876aebf55 /tidb/cdc/default/default/upstream/7374685442892162163 {"id":7374685442892162163,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.23850.out cli changefeed create --start-ts=450115081909829633 '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-attributes-test-30908?protocol=open-protocol&partition-num=4&kafka-version=2.4.1' Create changefeed successfully! ID: 9a67f4cc-36a8-4b89-8214-60136ec983bb Info: {"upstream_id":7374685442892162163,"namespace":"default","id":"9a67f4cc-36a8-4b89-8214-60136ec983bb","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-attributes-test-30908?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1","create_time":"2024-05-30T15:06:18.481907729+08:00","start_ts":450115081909829633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115081909829633,"checkpoint_ts":450115081909829633,"checkpoint_time":"2024-05-30 15:06:12.178"} PASS =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff successfully wait process cdc.test exit for 1-th time... + set +x [Thu May 30 15:06:20 CST 2024] <<<<<< START kafka consumer in ddl_attributes case >>>>>> wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2148bb880012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:19372, start at 2024-05-30 15:06:20.291783563 +0800 CST m=+6.041464945 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:20.302 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:20.308 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:20.308 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "mysql.db"="changefeed_auto_stop_1" "readallfields"="true" "mysql.host"="127.0.0.1" "mysql.port"="4000" "workload"="core" "insertproportion"="0" "operationcount"="0" "updateproportion"="0" "requestdistribution"="uniform" "mysql.user"="root" "readproportion"="0" "recordcount"="20" "scanproportion"="0" "threadcount"="4" "dotransactions"="false" ********************************************** Run finished, takes 15.851516ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3233.3, Avg(us): 3050, Min(us): 1033, Max(us): 9653, 95th(us): 10000, 99th(us): 10000 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:06:20 CST 2024] <<<<<< run test case generate_column success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/canal_json_basic Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... ***************** properties ***************** "dotransactions"="false" "readallfields"="true" "recordcount"="20" "mysql.port"="4000" "insertproportion"="0" "threadcount"="4" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "operationcount"="0" "updateproportion"="0" "mysql.db"="changefeed_auto_stop_2" "mysql.user"="root" "scanproportion"="0" "readproportion"="0" "workload"="core" ********************************************** Run finished, takes 16.722212ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2080.6, Avg(us): 3238, Min(us): 1261, Max(us): 7335, 95th(us): 8000, 99th(us): 8000 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2148bb880012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:19372, start at 2024-05-30 15:06:20.291783563 +0800 CST m=+6.041464945 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:20.302 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:20.308 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:20.308 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2148bd30000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:19458, start at 2024-05-30 15:06:20.376561586 +0800 CST m=+5.968741349 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:20.386 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:20.364 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:20.364 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ***************** properties ***************** "dotransactions"="false" "recordcount"="20" "updateproportion"="0" "operationcount"="0" "workload"="core" "readproportion"="0" "readallfields"="true" "requestdistribution"="uniform" "insertproportion"="0" "mysql.user"="root" "mysql.port"="4000" "threadcount"="4" "mysql.db"="changefeed_auto_stop_3" "scanproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 12.140532ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3306.9, Avg(us): 2201, Min(us): 1012, Max(us): 5709, 95th(us): 6000, 99th(us): 6000 Verifying downstream PD is started... ***************** properties ***************** "mysql.user"="root" "scanproportion"="0" "mysql.db"="changefeed_auto_stop_4" "updateproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "recordcount"="20" "requestdistribution"="uniform" "mysql.port"="4000" "dotransactions"="false" "readallfields"="true" "readproportion"="0" "insertproportion"="0" "workload"="core" "threadcount"="4" ********************************************** Run finished, takes 20.536746ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1464.6, Avg(us): 3154, Min(us): 997, Max(us): 8633, 95th(us): 9000, 99th(us): 9000 [Thu May 30 15:06:23 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1096510967.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.20701.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:06:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 {"id":"2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ebcb3e 2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 /tidb/cdc/default/default/upstream/7374685466538853897 {"id":7374685466538853897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 {"id":"2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ebcb3e 2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 /tidb/cdc/default/default/upstream/7374685466538853897 {"id":7374685466538853897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 {"id":"2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ebcb3e 2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 /tidb/cdc/default/default/upstream/7374685466538853897 {"id":7374685466538853897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Thu May 30 15:06:26 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1101711019.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450115086020247553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115086020247553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:06:29 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.2073820740.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:06:29 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c2c9e5b-2af5-4986-a30a-1678b8071f5c {"id":"1c2c9e5b-2af5-4986-a30a-1678b8071f5c","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 {"id":"2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ebcb3e 2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ebcb6f 1c2c9e5b-2af5-4986-a30a-1678b8071f5c /tidb/cdc/default/default/upstream/7374685466538853897 {"id":7374685466538853897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c2c9e5b-2af5-4986-a30a-1678b8071f5c {"id":"1c2c9e5b-2af5-4986-a30a-1678b8071f5c","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 {"id":"2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ebcb3e 2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ebcb6f 1c2c9e5b-2af5-4986-a30a-1678b8071f5c /tidb/cdc/default/default/upstream/7374685466538853897 {"id":7374685466538853897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c2c9e5b-2af5-4986-a30a-1678b8071f5c {"id":"1c2c9e5b-2af5-4986-a30a-1678b8071f5c","address":"127.0.0.1:8302","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/capture/2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 {"id":"2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2","address":"127.0.0.1:8301","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ebcb3e 2e03329a-ab7a-4cda-ac2c-c35fb5ead4e2 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ebcb6f 1c2c9e5b-2af5-4986-a30a-1678b8071f5c /tidb/cdc/default/default/upstream/7374685466538853897 {"id":7374685466538853897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Thu May 30 15:06:30 CST 2024] <<<<<< START kafka consumer in changefeed_auto_stop case >>>>>> check_changefeed_state http://127.0.0.1:2379 ae2d3449-e82c-4468-9235-c25e702a5f93 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=ae2d3449-e82c-4468-9235-c25e702a5f93 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c ae2d3449-e82c-4468-9235-c25e702a5f93 -s + info='{ "upstream_id": 7374685466538853897, "namespace": "default", "id": "ae2d3449-e82c-4468-9235-c25e702a5f93", "state": "normal", "checkpoint_tso": 450115083787042820, "checkpoint_time": "2024-05-30 15:06:19.339", "error": null }' + echo '{ "upstream_id": 7374685466538853897, "namespace": "default", "id": "ae2d3449-e82c-4468-9235-c25e702a5f93", "state": "normal", "checkpoint_tso": 450115083787042820, "checkpoint_time": "2024-05-30 15:06:19.339", "error": null }' { "upstream_id": 7374685466538853897, "namespace": "default", "id": "ae2d3449-e82c-4468-9235-c25e702a5f93", "state": "normal", "checkpoint_tso": 450115083787042820, "checkpoint_time": "2024-05-30 15:06:19.339", "error": null } ++ echo '{' '"upstream_id":' 7374685466538853897, '"namespace":' '"default",' '"id":' '"ae2d3449-e82c-4468-9235-c25e702a5f93",' '"state":' '"normal",' '"checkpoint_tso":' 450115083787042820, '"checkpoint_time":' '"2024-05-30' '15:06:19.339",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374685466538853897, '"namespace":' '"default",' '"id":' '"ae2d3449-e82c-4468-9235-c25e702a5f93",' '"state":' '"normal",' '"checkpoint_tso":' 450115083787042820, '"checkpoint_time":' '"2024-05-30' '15:06:19.339",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:06:32 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/adddcd2f-3fa3-47c5-84dc-2285e782ecab {"id":"adddcd2f-3fa3-47c5-84dc-2285e782ecab","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ff9ae6 adddcd2f-3fa3-47c5-84dc-2285e782ecab /tidb/cdc/default/default/upstream/7374685499383732645 {"id":7374685499383732645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/adddcd2f-3fa3-47c5-84dc-2285e782ecab {"id":"adddcd2f-3fa3-47c5-84dc-2285e782ecab","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ff9ae6 adddcd2f-3fa3-47c5-84dc-2285e782ecab /tidb/cdc/default/default/upstream/7374685499383732645 {"id":7374685499383732645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/adddcd2f-3fa3-47c5-84dc-2285e782ecab {"id":"adddcd2f-3fa3-47c5-84dc-2285e782ecab","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc851ff9ae6 adddcd2f-3fa3-47c5-84dc-2285e782ecab /tidb/cdc/default/default/upstream/7374685499383732645 {"id":7374685499383732645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.20787.out cli changefeed create --start-ts=450115086020247553 '--sink-uri=kafka+ssl://127.0.0.1:9092/ticdc-simple-test-9691?protocol=open-protocol&partition-num=4&kafka-client-id=cdc_test_simple&kafka-version=2.4.1&max-message-bytes=10485760' \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache table changefeed_auto_stop_1.usertable not exists for 2-th check, retry later [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container Create changefeed successfully! ID: c91abc37-2b80-4b9a-b303-e1f697f106d5 Info: {"upstream_id":7374685499383732645,"namespace":"default","id":"c91abc37-2b80-4b9a-b303-e1f697f106d5","sink_uri":"kafka+ssl://127.0.0.1:9092/ticdc-simple-test-9691?protocol=open-protocol\u0026partition-num=4\u0026kafka-client-id=cdc_test_simple\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:06:33.21611529+08:00","start_ts":450115086020247553,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115086020247553,"checkpoint_ts":450115086020247553,"checkpoint_time":"2024-05-30 15:06:27.858"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214974b40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:19386, start at 2024-05-30 15:06:32.114269981 +0800 CST m=+6.135504181 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:32.123 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:32.109 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:32.109 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214974b40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:19386, start at 2024-05-30 15:06:32.114269981 +0800 CST m=+6.135504181 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:32.123 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:32.109 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:32.109 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214975a00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:19465, start at 2024-05-30 15:06:32.213892512 +0800 CST m=+6.123486489 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:08:32.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:06:32.219 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:56:32.219 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x [Thu May 30 15:06:34 CST 2024] <<<<<< START kafka consumer in simple case >>>>>> succeed to verify meta placement rules table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later [Thu May 30 15:06:36 CST 2024] <<<<<< START cdc server in canal_json_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.2068520687.out server --log-file /tmp/tidb_cdc_test/canal_json_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 2-th time, retry later table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later table ddl_attributes.finish_mark not exists for 2-th check, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:06:39 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ea40f7b-c50d-4c1b-9976-46740a7dd6e0 {"id":"9ea40f7b-c50d-4c1b-9976-46740a7dd6e0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85232f7c1 9ea40f7b-c50d-4c1b-9976-46740a7dd6e0 /tidb/cdc/default/default/upstream/7374685541819812853 {"id":7374685541819812853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ea40f7b-c50d-4c1b-9976-46740a7dd6e0 {"id":"9ea40f7b-c50d-4c1b-9976-46740a7dd6e0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85232f7c1 9ea40f7b-c50d-4c1b-9976-46740a7dd6e0 /tidb/cdc/default/default/upstream/7374685541819812853 {"id":7374685541819812853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ea40f7b-c50d-4c1b-9976-46740a7dd6e0 {"id":"9ea40f7b-c50d-4c1b-9976-46740a7dd6e0","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85232f7c1 9ea40f7b-c50d-4c1b-9976-46740a7dd6e0 /tidb/cdc/default/default/upstream/7374685541819812853 {"id":7374685541819812853,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.cli.20733.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-canal-json-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' Create changefeed successfully! ID: dde2e8fb-a025-4e84-abaa-954874ec04ac Info: {"upstream_id":7374685541819812853,"namespace":"default","id":"dde2e8fb-a025-4e84-abaa-954874ec04ac","sink_uri":"kafka://127.0.0.1:9092/ticdc-canal-json-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-05-30T15:06:39.959644112+08:00","start_ts":450115089153130499,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115089153130499,"checkpoint_ts":450115089153130499,"checkpoint_time":"2024-05-30 15:06:39.809"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table changefeed_auto_stop_4.usertable exists check diff successfully table ddl_attributes.finish_mark not exists for 3-th check, retry later check data successfully wait process cdc.test exit for 1-th time... + set +x wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... wait process cdc.test exit for 3-th time... table ddl_attributes.finish_mark not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:06:42 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:06:43 CST 2024] <<<<<< run test case simple success! >>>>>> table ddl_attributes.finish_mark not exists for 5-th check, retry later [Thu May 30 15:06:46 CST 2024] <<<<<< START kafka consumer in canal_json_basic case >>>>>> table ddl_attributes.finish_mark not exists for 6-th check, retry later table ddl_attributes.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:06:52 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table test.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... table test.finish_mark not exists for 5-th check, retry later table test.finish_mark not exists for 6-th check, retry later Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore table test.finish_mark not exists for 7-th check, retry later Starting Downstream TiDB... table test.finish_mark not exists for 8-th check, retry later Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 78e03e07-f047-4e65-aa47-80b5f4759bd6 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=78e03e07-f047-4e65-aa47-80b5f4759bd6 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 78e03e07-f047-4e65-aa47-80b5f4759bd6 -s + info='{ "upstream_id": 7374685287633591945, "namespace": "default", "id": "78e03e07-f047-4e65-aa47-80b5f4759bd6", "state": "finished", "checkpoint_tso": 450115096794890251, "checkpoint_time": "2024-05-30 15:07:08.960", "error": null }' + echo '{ "upstream_id": 7374685287633591945, "namespace": "default", "id": "78e03e07-f047-4e65-aa47-80b5f4759bd6", "state": "finished", "checkpoint_tso": 450115096794890251, "checkpoint_time": "2024-05-30 15:07:08.960", "error": null }' { "upstream_id": 7374685287633591945, "namespace": "default", "id": "78e03e07-f047-4e65-aa47-80b5f4759bd6", "state": "finished", "checkpoint_tso": 450115096794890251, "checkpoint_time": "2024-05-30 15:07:08.960", "error": null } ++ echo '{' '"upstream_id":' 7374685287633591945, '"namespace":' '"default",' '"id":' '"78e03e07-f047-4e65-aa47-80b5f4759bd6",' '"state":' '"finished",' '"checkpoint_tso":' 450115096794890251, '"checkpoint_time":' '"2024-05-30' '15:07:08.960",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7374685287633591945, '"namespace":' '"default",' '"id":' '"78e03e07-f047-4e65-aa47-80b5f4759bd6",' '"state":' '"finished",' '"checkpoint_tso":' 450115096794890251, '"checkpoint_time":' '"2024-05-30' '15:07:08.960",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:07:15 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> table test.finish_mark not exists for 10-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 11-th check, retry later table test.finish_mark exists check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214c95e00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:21865, start at 2024-05-30 15:07:23.407909586 +0800 CST m=+8.911029171 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:09:23.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:07:23.384 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:57:23.384 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214c95e00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:21865, start at 2024-05-30 15:07:23.407909586 +0800 CST m=+8.911029171 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:09:23.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:07:23.384 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:57:23.384 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214cbc280018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:21923, start at 2024-05-30 15:07:25.884534407 +0800 CST m=+8.389498916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:09:25.913 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:07:25.884 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:57:25.884 +0800 All versions after safe point can be accessed. (DO NOT EDIT) start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.23120.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x + tso='450115102987517953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115102987517953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Thu May 30 15:07:34 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2316023162.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:07:37 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2229240-ea80-4140-9118-d10db4000b4a {"id":"d2229240-ea80-4140-9118-d10db4000b4a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc852d721f8 d2229240-ea80-4140-9118-d10db4000b4a /tidb/cdc/default/default/upstream/7374685730319292843 {"id":7374685730319292843,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2229240-ea80-4140-9118-d10db4000b4a {"id":"d2229240-ea80-4140-9118-d10db4000b4a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc852d721f8 d2229240-ea80-4140-9118-d10db4000b4a /tidb/cdc/default/default/upstream/7374685730319292843 {"id":7374685730319292843,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2229240-ea80-4140-9118-d10db4000b4a {"id":"d2229240-ea80-4140-9118-d10db4000b4a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc852d721f8 d2229240-ea80-4140-9118-d10db4000b4a /tidb/cdc/default/default/upstream/7374685730319292843 {"id":7374685730319292843,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214db28c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf, pid:6480, start at 2024-05-30 15:07:41.616886964 +0800 CST m=+6.167850856 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:09:41.623 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:07:41.603 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:57:41.603 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214db28c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf, pid:6480, start at 2024-05-30 15:07:41.616886964 +0800 CST m=+6.167850856 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:09:41.623 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:07:41.603 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:57:41.603 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214db2c40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-530gf, pid:6563, start at 2024-05-30 15:07:41.619718485 +0800 CST m=+6.075925566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:09:41.626 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:07:41.617 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:57:41.617 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 6-th check, retry later [Thu May 30 15:07:45 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.79197921.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/30 15:07:44.852 +08:00] [WARN] [diff.go:765] ["rows is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [cost=1m34.697455411s] [2024/05/30 15:07:44.855 +08:00] [WARN] [diff.go:383] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:07:49 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b8dd8ef-74d4-4d55-a4f8-961073e00344 {"id":"1b8dd8ef-74d4-4d55-a4f8-961073e00344","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8533e87bc 1b8dd8ef-74d4-4d55-a4f8-961073e00344 /tidb/cdc/default/default/upstream/7374685841106449002 {"id":7374685841106449002,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b8dd8ef-74d4-4d55-a4f8-961073e00344 {"id":"1b8dd8ef-74d4-4d55-a4f8-961073e00344","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8533e87bc 1b8dd8ef-74d4-4d55-a4f8-961073e00344 /tidb/cdc/default/default/upstream/7374685841106449002 + grep -q 'failed to get info:' {"id":7374685841106449002,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b8dd8ef-74d4-4d55-a4f8-961073e00344 {"id":"1b8dd8ef-74d4-4d55-a4f8-961073e00344","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8533e87bc 1b8dd8ef-74d4-4d55-a4f8-961073e00344 /tidb/cdc/default/default/upstream/7374685841106449002 {"id":7374685841106449002,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 4aa8875b-2c3e-420f-8857-97371df23c5e Info: {"upstream_id":7374685841106449002,"namespace":"default","id":"4aa8875b-2c3e-420f-8857-97371df23c5e","sink_uri":"kafka://127.0.0.1:9092/ticdc-force_replicate_table-test-27101?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:07:49.27638077+08:00","start_ts":450115106479276033,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115106479276033,"checkpoint_ts":450115106479276033,"checkpoint_time":"2024-05-30 15:07:45.903"} [Thu May 30 15:07:49 CST 2024] <<<<<< START kafka consumer in force_replicate_table case >>>>>> consumer replica config found: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/force_replicate_table/conf/changefeed.toml cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:07:49 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> [2024/05/30 15:07:50.900 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:07:53 CST 2024] <<<<<< run test case cdc success! >>>>>> table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Thu May 30 15:07:57 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2324623248.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ true != \n\o ]] + set +x table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_topics Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=7,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2796 0 --:--:-- --:--:-- --:--:-- 2833 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-30 15:07:58.025","puller_resolved_ts":"2024-05-30 15:07:50.025","last_synced_ts":"2024-05-30 15:05:41.225","now_ts":"2024-05-30 15:07:58.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:07:58.025","puller_resolved_ts":"2024-05-30' '15:07:50.025","last_synced_ts":"2024-05-30' '15:05:41.225","now_ts":"2024-05-30' '15:07:58.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 14761 10.1 0.0 13796828 145404 ? Sl 15:05 0:16 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 14825 6.7 0.0 13804960 135456 ? Sl 15:05 0:10 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1717052884.6368937,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014e4540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717052884.6369581,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717052884.8033473,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0031f9880/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1717052884.8034298,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717052885.142156,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002363180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717052885.1422744,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-30T15:08:08.892758+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d30e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:08.900832+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cac8c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:08.999604+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e26e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:08:14 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214fba8c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:21867, start at 2024-05-30 15:08:14.930345433 +0800 CST m=+5.752464153 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:10:14.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:08:14.934 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:58:14.934 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) {"level":"warn","ts":"2024-05-30T15:08:14.893731+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d30e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:14.902543+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cac8c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:15.000527+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e26e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Thu May 30 15:08:17 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214fba8c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:21867, start at 2024-05-30 15:08:14.930345433 +0800 CST m=+5.752464153 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:10:14.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:08:14.934 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:58:14.934 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f214fbc0c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:21941, start at 2024-05-30 15:08:15.025565285 +0800 CST m=+5.740307291 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:10:15.033 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:08:15.029 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:58:15.029 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_topics/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_topics/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.cli.23230.out cli tso query --pd=http://127.0.0.1:2379 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:08:20.89492+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d30e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:20.903952+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cac8c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:21.001684+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e26e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215014000012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k, pid:21783, start at 2024-05-30 15:08:20.640439977 +0800 CST m=+5.814320619 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:10:20.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:08:20.657 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:58:20.657 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450115115386929157 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115115386929157 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:08:21 CST 2024] <<<<<< START cdc server in multi_topics case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.2325523257.out server --log-file /tmp/tidb_cdc_test/multi_topics/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_topics/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215014000012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k, pid:21783, start at 2024-05-30 15:08:20.640439977 +0800 CST m=+5.814320619 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:10:20.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:08:20.657 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:58:20.657 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215015880009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-7vh0k, pid:21863, start at 2024-05-30 15:08:20.717683436 +0800 CST m=+5.780070542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:10:20.726 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:08:20.706 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:58:20.706 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:08:24 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f57eb7f-e52d-43f2-b40a-cbdb475e2c64 {"id":"8f57eb7f-e52d-43f2-b40a-cbdb475e2c64","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc853c439c5 8f57eb7f-e52d-43f2-b40a-cbdb475e2c64 /tidb/cdc/default/default/upstream/7374685989670107105 {"id":7374685989670107105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f57eb7f-e52d-43f2-b40a-cbdb475e2c64 {"id":"8f57eb7f-e52d-43f2-b40a-cbdb475e2c64","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc853c439c5 8f57eb7f-e52d-43f2-b40a-cbdb475e2c64 /tidb/cdc/default/default/upstream/7374685989670107105 {"id":7374685989670107105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f57eb7f-e52d-43f2-b40a-cbdb475e2c64 {"id":"8f57eb7f-e52d-43f2-b40a-cbdb475e2c64","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc853c439c5 8f57eb7f-e52d-43f2-b40a-cbdb475e2c64 /tidb/cdc/default/default/upstream/7374685989670107105 {"id":7374685989670107105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.cli.23309.out cli changefeed create --start-ts=450115115386929157 '--sink-uri=kafka://127.0.0.1:9092/multi_topics?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics/conf/changefeed.toml Create changefeed successfully! ID: 1ea39ad8-10ee-436f-adc4-5eef3b1116a2 Info: {"upstream_id":7374685989670107105,"namespace":"default","id":"1ea39ad8-10ee-436f-adc4-5eef3b1116a2","sink_uri":"kafka://127.0.0.1:9092/multi_topics?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-05-30T15:08:24.864317775+08:00","start_ts":450115115386929157,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["workload.*"],"topic":"workload"},{"matcher":["test.*"],"topic":"{schema}_{table}"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115115386929157,"checkpoint_ts":450115115386929157,"checkpoint_time":"2024-05-30 15:08:19.883"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.23104.out cli tso query --pd=http://127.0.0.1:2379 + set +x \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:08:26.895883+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d30e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:26.905641+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cac8c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:27.003241+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e26e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [Pipeline] // dir [Pipeline] } + set +x + tso='450115116913917953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115116913917953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Thu May 30 15:08:27 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.2314623148.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:08:28.88389+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d30e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-30T15:08:28.883954+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-30T15:08:28.888499+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cac8c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-30T15:08:28.888565+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-30T15:08:28.988253+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e26e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-30T15:08:28.988307+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:08:30 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fcb82fd1-6421-4495-8f52-50ad0126f67f {"id":"fcb82fd1-6421-4495-8f52-50ad0126f67f","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc853de4dea fcb82fd1-6421-4495-8f52-50ad0126f67f /tidb/cdc/default/default/upstream/7374686014020565821 {"id":7374686014020565821,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fcb82fd1-6421-4495-8f52-50ad0126f67f {"id":"fcb82fd1-6421-4495-8f52-50ad0126f67f","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc853de4dea fcb82fd1-6421-4495-8f52-50ad0126f67f /tidb/cdc/default/default/upstream/7374686014020565821 {"id":7374686014020565821,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fcb82fd1-6421-4495-8f52-50ad0126f67f {"id":"fcb82fd1-6421-4495-8f52-50ad0126f67f","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc853de4dea fcb82fd1-6421-4495-8f52-50ad0126f67f /tidb/cdc/default/default/upstream/7374686014020565821 {"id":7374686014020565821,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.23194.out cli changefeed create --start-ts=450115116913917953 '--sink-uri=kafka://127.0.0.1:9092/ticdc-split-region-test-6267?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: b91a7224-a196-483d-9f91-c62ff6c714ee Info: {"upstream_id":7374686014020565821,"namespace":"default","id":"b91a7224-a196-483d-9f91-c62ff6c714ee","sink_uri":"kafka://127.0.0.1:9092/ticdc-split-region-test-6267?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:08:30.980359915+08:00","start_ts":450115116913917953,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115116913917953,"checkpoint_ts":450115116913917953,"checkpoint_time":"2024-05-30 15:08:25.708"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Thu May 30 15:08:32 CST 2024] <<<<<< START kafka consumer in split_region case >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table split_region.test1 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:08:32.896858+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d30e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:32.906297+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cac8c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:33.004704+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e26e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table split_region.test1 not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table split_region.test1 exists table split_region.test2 exists check diff failed 1-th time, retry later Verifying downstream PD is started... check diff successfully 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:08:38.898776+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d30e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:38.907901+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cac8c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:39.005631+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e26e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [2024/05/30 15:08:32.851 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2024/05/30 15:08:33.043 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/30 15:08:33.051 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/30 15:08:33.209 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/30 15:08:33.210 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/30 15:08:33.210 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/30 15:08:33.212 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/30 15:08:33.398 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/30 15:08:33.400 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/30 15:08:34.588 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/30 15:08:34.589 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/30 15:08:34.605 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/30 15:08:34.611 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/30 15:08:34.763 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/30 15:08:34.809 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/30 15:08:34.938 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/30 15:08:34.938 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/30 15:08:34.980 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/30 15:08:34.985 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/30 15:08:35.109 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/30 15:08:36.188 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/30 15:08:37.848 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/30 15:08:37.853 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1717052919.637946,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014e4540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717052919.63801,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717052919.8039103,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0031f9880/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1717052919.803969,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717052920.1428964,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002363180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717052920.142973,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/30 15:08:43.363 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... [2024/05/30 15:08:43.558 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/30 15:08:43.570 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/30 15:08:43.713 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/30 15:08:43.715 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:08:43.750 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/30 15:08:43.751 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/30 15:08:43.896 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/30 15:08:43.968 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/30 15:08:44.067 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/30 15:08:44.068 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/30 15:08:44.150 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/30 15:08:44.153 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/30 15:08:44.222 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/30 15:08:44.334 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/30 15:08:44.393 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/30 15:08:44.395 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/30 15:08:44.532 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/30 15:08:44.539 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/30 15:08:44.582 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/30 15:08:44.717 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/30 15:08:44.742 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/30 15:08:44.744 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:08:44.899594+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d30e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:08:44.909151+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cac8c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} [2024/05/30 15:08:44.924 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/30 15:08:44.929 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/30 15:08:44.932 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] {"level":"warn","ts":"2024-05-30T15:08:45.006636+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e26e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [2024/05/30 15:08:45.084 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/30 15:08:45.086 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/30 15:08:45.105 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/30 15:08:45.268 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/30 15:08:45.277 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/30 15:08:45.283 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/30 15:08:45.444 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/30 15:08:45.445 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/30 15:08:45.446 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:08:45.605 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/30 15:08:45.612 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/30 15:08:45.614 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/30 15:08:45.822 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/30 15:08:45.832 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/30 15:08:45.837 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/30 15:08:45.985 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/30 15:08:45.992 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/30 15:08:46.000 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/30 15:08:46.161 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/30 15:08:46.187 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/30 15:08:46.193 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/30 15:08:46.338 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/30 15:08:46.346 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/30 15:08:46.426 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/30 15:08:46.508 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/30 15:08:46.585 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/30 15:08:46.594 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/30 15:08:46.725 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/30 15:08:46.735 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/30 15:08:46.798 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/30 15:08:46.902 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/30 15:08:46.968 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/30 15:08:46.981 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/30 15:08:47.090 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/30 15:08:47.100 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/30 15:08:47.140 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/30 15:08:47.268 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/30 15:08:47.480 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/30 15:08:47.491 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:08:47.660 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/30 15:08:47.833 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/30 15:08:47.843 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/30 15:08:48.053 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/30 15:08:48.241 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/30 15:08:48.251 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2151d6ec0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:24166, start at 2024-05-30 15:08:49.49365068 +0800 CST m=+6.081062102 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:10:49.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:08:49.467 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:58:49.467 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2151d6ec0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:24166, start at 2024-05-30 15:08:49.49365068 +0800 CST m=+6.081062102 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:10:49.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:08:49.467 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:58:49.467 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2151d8800014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-smw3b, pid:24240, start at 2024-05-30 15:08:49.614290889 +0800 CST m=+6.013966076 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:10:49.621 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:08:49.618 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:58:49.618 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later [Thu May 30 15:08:53 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.2546825470.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:08:57 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e9d47ad-6f75-4b41-8ff1-4e5d469a4388 {"id":"9e9d47ad-6f75-4b41-8ff1-4e5d469a4388","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8543c14c8 9e9d47ad-6f75-4b41-8ff1-4e5d469a4388 /tidb/cdc/default/default/upstream/7374686120904376999 {"id":7374686120904376999,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e9d47ad-6f75-4b41-8ff1-4e5d469a4388 {"id":"9e9d47ad-6f75-4b41-8ff1-4e5d469a4388","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8543c14c8 9e9d47ad-6f75-4b41-8ff1-4e5d469a4388 /tidb/cdc/default/default/upstream/7374686120904376999 {"id":7374686120904376999,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e9d47ad-6f75-4b41-8ff1-4e5d469a4388 {"id":"9e9d47ad-6f75-4b41-8ff1-4e5d469a4388","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8543c14c8 9e9d47ad-6f75-4b41-8ff1-4e5d469a4388 /tidb/cdc/default/default/upstream/7374686120904376999 {"id":7374686120904376999,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.25515.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-event-filter-10000?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml [2024/05/30 15:08:56.161 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2024/05/30 15:08:56.343 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/30 15:08:56.358 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/30 15:08:56.494 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/30 15:08:56.496 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/30 15:08:56.527 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/30 15:08:56.529 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/30 15:08:56.650 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/30 15:08:56.688 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/30 15:08:56.802 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/30 15:08:56.804 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/30 15:08:56.837 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/30 15:08:56.838 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/30 15:08:56.961 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/30 15:08:56.991 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/30 15:08:57.110 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/30 15:08:57.112 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/30 15:08:57.140 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/30 15:08:57.142 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:08:57 CST 2024] <<<<<< run test case split_region success! >>>>>> [2024/05/30 15:08:57.262 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/30 15:08:57.291 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] Create changefeed successfully! ID: e714bedd-a7d8-4f98-8e10-110cdf949ff9 Info: {"upstream_id":7374686120904376999,"namespace":"default","id":"e714bedd-a7d8-4f98-8e10-110cdf949ff9","sink_uri":"kafka://127.0.0.1:9092/ticdc-event-filter-10000?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:08:57.556237512+08:00","start_ts":450115125213134852,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115125213134852,"checkpoint_ts":450115125213134852,"checkpoint_time":"2024-05-30 15:08:57.367"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/05/30 15:08:57.433 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/30 15:08:57.434 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/30 15:08:57.503 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/30 15:08:57.505 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/30 15:08:57.735 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/30 15:08:57.767 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/30 15:08:57.903 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/30 15:08:57.904 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/30 15:08:57.926 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/30 15:08:57.928 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/30 15:08:58.206 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/30 15:08:58.234 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/30 15:08:58.470 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/30 15:08:58.472 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/30 15:08:58.497 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/30 15:08:58.499 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/30 15:08:58.632 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/30 15:08:58.659 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/30 15:08:58.800 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/30 15:08:58.801 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/30 15:08:58.838 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/30 15:08:58.840 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/30 15:08:58.956 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory [2024/05/30 15:08:59.000 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/30 15:08:59.117 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/30 15:08:59.118 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/30 15:08:59.163 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/30 15:08:59.165 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/30 15:08:59.279 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/30 15:08:59.334 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] + set +x [Thu May 30 15:08:59 CST 2024] <<<<<< START kafka consumer in event_filter case >>>>>> [2024/05/30 15:08:59.516 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/30 15:08:59.517 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/30 15:08:59.633 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/30 15:08:59.635 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/30 15:08:59.820 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/30 15:08:59.869 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/30 15:08:59.997 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/30 15:08:59.998 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] table event_filter.t1 does not exists table event_filter.t1 not exists for 1-th check, retry later [2024/05/30 15:09:00.050 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/30 15:09:00.052 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/30 15:09:00.295 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/30 15:09:00.409 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/30 15:09:00.624 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/30 15:09:00.626 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/30 15:09:00.745 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/30 15:09:00.747 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/30 15:09:00.862 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/30 15:09:00.946 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/30 15:09:01.047 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/30 15:09:01.048 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/30 15:09:01.142 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/30 15:09:01.143 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table event_filter.t_normal exists table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table event_filter.t_alter exists table event_filter.finish_mark exists check diff failed 1-th time, retry later [2024/05/30 15:09:07.465 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2024/05/30 15:09:07.743 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/30 15:09:07.750 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/30 15:09:08.017 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/30 15:09:08.020 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:09:08.107 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/30 15:09:08.108 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/30 15:09:08.342 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/30 15:09:08.423 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/30 15:09:08.550 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/30 15:09:08.552 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/30 15:09:08.700 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/30 15:09:08.701 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/30 15:09:08.915 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/30 15:09:09.021 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/30 15:09:09.208 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/30 15:09:09.209 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/30 15:09:09.398 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/30 15:09:09.399 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/30 15:09:09.543 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/30 15:09:09.670 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/30 15:09:09.803 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/30 15:09:09.804 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/30 15:09:10.004 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/30 15:09:10.006 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] check diff failed 2-th time, retry later [2024/05/30 15:09:10.142 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/30 15:09:10.278 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/30 15:09:10.354 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/30 15:09:10.356 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/30 15:09:10.495 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/30 15:09:10.496 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/30 15:09:10.551 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:09:10.737 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/30 15:09:10.783 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/30 15:09:10.795 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/30 15:09:10.912 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/30 15:09:10.913 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/30 15:09:10.954 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/30 15:09:11.132 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/30 15:09:11.201 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/30 15:09:11.204 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/30 15:09:11.355 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/30 15:09:11.355 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/30 15:09:11.417 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/30 15:09:11.532 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/30 15:09:11.575 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/30 15:09:11.576 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/30 15:09:11.734 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/30 15:09:11.735 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/30 15:09:11.773 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] check diff failed 3-th time, retry later [2024/05/30 15:09:11.926 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/30 15:09:11.958 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/30 15:09:11.959 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/30 15:09:12.085 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/30 15:09:12.086 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/30 15:09:12.106 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/30 15:09:12.237 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/30 15:09:12.249 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/30 15:09:12.251 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/30 15:09:12.394 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/30 15:09:12.395 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/30 15:09:12.411 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:09:15 CST 2024] <<<<<< run test case event_filter success! >>>>>> [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215356280012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:17998, start at 2024-05-30 15:09:14.020605606 +0800 CST m=+6.117332864 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:11:14.028 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:09:13.994 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:59:13.994 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215356280012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:17998, start at 2024-05-30 15:09:14.020605606 +0800 CST m=+6.117332864 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:11:14.028 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:09:13.994 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:59:13.994 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215357bc000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:18064, start at 2024-05-30 15:09:14.114124391 +0800 CST m=+6.042519465 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:11:14.123 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:09:14.095 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-14:59:14.095 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19282.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/30 15:09:18.375 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2024/05/30 15:09:18.557 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/30 15:09:18.742 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/30 15:09:18.743 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/30 15:09:18.957 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/30 15:09:18.997 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/30 15:09:19.148 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/30 15:09:19.150 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/30 15:09:19.150 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/30 15:09:19.151 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/30 15:09:19.306 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/30 15:09:19.310 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/30 15:09:19.455 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/30 15:09:19.455 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/30 15:09:19.474 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/30 15:09:19.475 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/30 15:09:19.604 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/30 15:09:19.634 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/30 15:09:19.752 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/30 15:09:19.753 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/30 15:09:19.789 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/30 15:09:19.791 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/30 15:09:19.928 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/30 15:09:19.962 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/30 15:09:20.090 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/30 15:09:20.090 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/30 15:09:20.129 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/30 15:09:20.130 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] + set +x + tso='450115130804142081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115130804142081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450115130804142081 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Thu May 30 15:09:20 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1931519317.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/30 15:09:20.242 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/30 15:09:20.296 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/30 15:09:20.406 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/30 15:09:20.407 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/30 15:09:20.500 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/30 15:09:20.502 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/30 15:09:20.571 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/30 15:09:20.667 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/30 15:09:20.738 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/30 15:09:20.739 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/30 15:09:20.857 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/30 15:09:20.859 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/30 15:09:20.929 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/30 15:09:21.112 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/30 15:09:21.167 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/30 15:09:21.168 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/30 15:09:21.270 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/30 15:09:21.271 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/30 15:09:21.329 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/30 15:09:21.425 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/30 15:09:21.486 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/30 15:09:21.486 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/30 15:09:22.023 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/30 15:09:22.025 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/30 15:09:22.134 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/30 15:09:22.258 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/30 15:09:22.335 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/30 15:09:22.336 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/30 15:09:22.475 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/30 15:09:22.477 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/30 15:09:22.525 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/30 15:09:22.676 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/30 15:09:22.715 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/30 15:09:22.716 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/30 15:09:22.871 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/30 15:09:22.874 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/30 15:09:22.912 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/30 15:09:23.122 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/30 15:09:23.141 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/30 15:09:23.142 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/30 15:09:23.278 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/30 15:09:23.280 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/30 15:09:23.310 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:09:23 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6bf7c0d-1c61-4075-8028-7ec25441917c {"id":"f6bf7c0d-1c61-4075-8028-7ec25441917c","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc854a5acc7 f6bf7c0d-1c61-4075-8028-7ec25441917c /tidb/cdc/default/default/upstream/7374686238421614898 {"id":7374686238421614898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6bf7c0d-1c61-4075-8028-7ec25441917c {"id":"f6bf7c0d-1c61-4075-8028-7ec25441917c","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc854a5acc7 f6bf7c0d-1c61-4075-8028-7ec25441917c /tidb/cdc/default/default/upstream/7374686238421614898 {"id":7374686238421614898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6bf7c0d-1c61-4075-8028-7ec25441917c {"id":"f6bf7c0d-1c61-4075-8028-7ec25441917c","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc854a5acc7 f6bf7c0d-1c61-4075-8028-7ec25441917c /tidb/cdc/default/default/upstream/7374686238421614898 {"id":7374686238421614898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450115130804142081 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19368.out cli changefeed create --start-ts=450115130804142081 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374686238421614898,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-30T15:09:23.733293229+08:00","start_ts":450115130804142081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115130804142081,"checkpoint_ts":450115130804142081,"checkpoint_time":"2024-05-30 15:09:18.695"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 [2024/05/30 15:09:28.949 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2024/05/30 15:09:29.125 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/30 15:09:29.149 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/30 15:09:29.277 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/30 15:09:29.278 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/30 15:09:29.334 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/30 15:09:29.336 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/30 15:09:29.460 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/30 15:09:29.546 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/30 15:09:29.718 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/30 15:09:29.719 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/30 15:09:29.860 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/30 15:09:29.862 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/30 15:09:29.983 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/30 15:09:30.068 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/30 15:09:30.216 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/30 15:09:30.217 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/30 15:09:30.333 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/30 15:09:30.336 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/30 15:09:30.596 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/30 15:09:30.726 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/30 15:09:30.997 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/30 15:09:30.998 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/30 15:09:31.112 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/30 15:09:31.115 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/30 15:09:31.365 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/30 15:09:31.519 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/30 15:09:31.753 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/30 15:09:31.755 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/30 15:09:31.855 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/30 15:09:31.858 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/30 15:09:32.113 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/30 15:09:32.351 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/30 15:09:32.731 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/30 15:09:32.732 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/30 15:09:32.932 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/30 15:09:32.934 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/30 15:09:33.325 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 17417 17.5 0.4 3679212 1659304 ? Sl 15:09 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 17418 17.5 0.4 3686892 1656932 ? Sl 15:09 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 17419 26.4 0.4 3714544 1730348 ? Sl 15:09 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 17421 25.2 0.4 3713516 1721300 ? Sl 15:09 0:06 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3171 0 --:--:-- --:--:-- --:--:-- 3197 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-30 15:09:32.095","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-30 15:09:25.695","now_ts":"2024-05-30 15:09:32.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:09:32.095","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:09:25.695","now_ts":"2024-05-30' '15:09:32.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:09:32.095","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:09:25.695","now_ts":"2024-05-30' '15:09:32.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 \033[0;36m<<< Run all test success >>>\033[0m [2024/05/30 15:09:33.525 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [2024/05/30 15:09:33.763 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/30 15:09:33.764 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [2024/05/30 15:09:45.433 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m12.582453022s"] table mark.finish_mark_0 exists table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 exists table mark.finish_mark_4 exists table mark.finish_mark not exists for 1-th check, retry later table mark.finish_mark not exists for 2-th check, retry later table mark.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:09:52 CST 2024] <<<<<< run test case multi_source success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Thu May 30 15:10:26 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 1 [Thu May 30 15:10:26 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 2 [Thu May 30 15:10:26 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 3 [Thu May 30 15:10:26 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> table test.table1 not exists for 1-th check, retry later table test.table1 not exists for 2-th check, retry later table test.table1 exists table test.table2 exists table test.table3 exists check diff successfully table test.table10 not exists for 1-th check, retry later table test.table10 exists table test.table20 exists check diff successfully [Thu May 30 15:11:04 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 10 [Thu May 30 15:11:04 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 20 [Thu May 30 15:11:04 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: finish table test.finish not exists for 1-th check, retry later table test.finish not exists for 2-th check, retry later table test.finish exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:11:10 CST 2024] <<<<<< run test case multi_topics success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: kafka... <<================= Starting schema registry... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8088 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8088 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:11:25 GMT < Content-Type: application/vnd.schemaregistry.v1+json < Vary: Accept-Encoding, User-Agent < Content-Length: 2 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/avro_basic Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215c4ac40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:24837, start at 2024-05-30 15:11:40.761837654 +0800 CST m=+5.979780050 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:13:40.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:11:40.771 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:01:40.771 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215c4ac40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:24837, start at 2024-05-30 15:11:40.761837654 +0800 CST m=+5.979780050 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:13:40.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:11:40.771 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:01:40.771 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215c4c2c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:24912, start at 2024-05-30 15:11:40.849595538 +0800 CST m=+5.962621539 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:13:40.857 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:11:40.860 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:01:40.860 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/avro_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/avro_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 51 49 0 0 100 25 0 322 --:--:-- --:--:-- --:--:-- 320 100 49 100 24 100 25 308 321 --:--:-- --:--:-- --:--:-- 320 {"compatibility":"NONE"}+ pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.cli.26135.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450115169319649281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115169319649281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:11:47 CST 2024] <<<<<< START cdc server in avro_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.2615926161.out server --log-file /tmp/tidb_cdc_test/avro_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/avro_basic/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:11:50 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1466bc3-ac60-4923-8a21-027592a287aa {"id":"a1466bc3-ac60-4923-8a21-027592a287aa","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc856e585bf a1466bc3-ac60-4923-8a21-027592a287aa /tidb/cdc/default/default/upstream/7374686875425841469 {"id":7374686875425841469,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1466bc3-ac60-4923-8a21-027592a287aa {"id":"a1466bc3-ac60-4923-8a21-027592a287aa","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc856e585bf a1466bc3-ac60-4923-8a21-027592a287aa /tidb/cdc/default/default/upstream/7374686875425841469 {"id":7374686875425841469,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1466bc3-ac60-4923-8a21-027592a287aa {"id":"a1466bc3-ac60-4923-8a21-027592a287aa","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc856e585bf a1466bc3-ac60-4923-8a21-027592a287aa /tidb/cdc/default/default/upstream/7374686875425841469 {"id":7374686875425841469,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.cli.26205.out cli changefeed create --start-ts=450115169319649281 '--sink-uri=kafka://127.0.0.1:9092/ticdc-avro-test?protocol=avro&enable-tidb-extension=true&avro-enable-watermark=true&avro-decimal-handling-mode=string&avro-bigint-unsigned-handling-mode=string' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/avro_basic/conf/changefeed.toml --schema-registry=http://127.0.0.1:8088 Create changefeed successfully! ID: c5c5de33-504a-4f93-a6c7-810e9a2f3303 Info: {"upstream_id":7374686875425841469,"namespace":"default","id":"c5c5de33-504a-4f93-a6c7-810e9a2f3303","sink_uri":"kafka://127.0.0.1:9092/ticdc-avro-test?protocol=avro\u0026enable-tidb-extension=true\u0026avro-enable-watermark=true\u0026avro-decimal-handling-mode=string\u0026avro-bigint-unsigned-handling-mode=string","create_time":"2024-05-30T15:11:50.656089764+08:00","start_ts":450115169319649281,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"avro","schema_registry":"http://127.0.0.1:8088","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115169319649281,"checkpoint_ts":450115169319649281,"checkpoint_time":"2024-05-30 15:11:45.620"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Thu May 30 15:11:52 CST 2024] <<<<<< START kafka consumer in avro_basic case >>>>>> schema registry uri found: http://127.0.0.1:8088 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 6558 0 --:--:-- --:--:-- --:--:-- 6572 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-30 15:09:32.095","puller_resolved_ts":"2024-05-30 15:09:32.095","last_synced_ts":"2024-05-30 15:09:25.695","now_ts":"2024-05-30 15:11:42.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:09:32.095","puller_resolved_ts":"2024-05-30' '15:09:32.095","last_synced_ts":"2024-05-30' '15:09:25.695","now_ts":"2024-05-30' '15:11:42.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:09:32.095","puller_resolved_ts":"2024-05-30' '15:09:32.095","last_synced_ts":"2024-05-30' '15:09:25.695","now_ts":"2024-05-30' '15:11:42.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... table test.finish_mark not exists for 1-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:12:00 CST 2024] <<<<<< run test case avro_basic success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215df1340001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:20469, start at 2024-05-30 15:12:07.758329469 +0800 CST m=+5.782321445 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:14:07.766 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:12:07.757 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:02:07.757 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215df1340001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:20469, start at 2024-05-30 15:12:07.758329469 +0800 CST m=+5.782321445 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:14:07.766 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:12:07.757 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:02:07.757 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215df3800008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:20541, start at 2024-05-30 15:12:07.914653526 +0800 CST m=+5.824780779 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:14:07.923 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:12:07.904 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:02:07.904 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.21755.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450115176420605953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115176420605953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450115176420605953 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Thu May 30 15:12:14 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2178121783.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/canal_json_handle_key_only Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:12:17 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e517414-5ded-46ed-bc3b-1089699e7bc5 {"id":"6e517414-5ded-46ed-bc3b-1089699e7bc5","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8574dcbbe 6e517414-5ded-46ed-bc3b-1089699e7bc5 /tidb/cdc/default/default/upstream/7374686989908135193 {"id":7374686989908135193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e517414-5ded-46ed-bc3b-1089699e7bc5 {"id":"6e517414-5ded-46ed-bc3b-1089699e7bc5","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8574dcbbe 6e517414-5ded-46ed-bc3b-1089699e7bc5 /tidb/cdc/default/default/upstream/7374686989908135193 {"id":7374686989908135193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e517414-5ded-46ed-bc3b-1089699e7bc5 {"id":"6e517414-5ded-46ed-bc3b-1089699e7bc5","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8574dcbbe 6e517414-5ded-46ed-bc3b-1089699e7bc5 /tidb/cdc/default/default/upstream/7374686989908135193 {"id":7374686989908135193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450115176420605953 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.21832.out cli changefeed create --start-ts=450115176420605953 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374686989908135193,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-30T15:12:17.700454028+08:00","start_ts":450115176420605953,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115176420605953,"checkpoint_ts":450115176420605953,"checkpoint_time":"2024-05-30 15:12:12.708"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t1 exists + sleep 5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215f0d340012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:27202, start at 2024-05-30 15:12:25.979666032 +0800 CST m=+5.888408559 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:14:25.990 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:12:25.983 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:02:25.983 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 20465 4.1 0.0 2452228 164112 ? Sl 15:12 0:01 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717053121966403564.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 20469 12.0 0.0 2417748 187816 ? Sl 15:12 0:03 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717053121969730098.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 20541 12.6 0.0 2445668 241880 ? Sl 15:12 0:03 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717053122081365765.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3921 0 --:--:-- --:--:-- --:--:-- 3983 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-30 15:12:26.108","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-30 15:12:19.658","now_ts":"2024-05-30 15:12:26.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:12:26.108","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:12:19.658","now_ts":"2024-05-30' '15:12:26.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:12:26.108","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:12:19.658","now_ts":"2024-05-30' '15:12:26.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215f0d340012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:27202, start at 2024-05-30 15:12:25.979666032 +0800 CST m=+5.888408559 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:14:25.990 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:12:25.983 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:02:25.983 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f215f0d24000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:27270, start at 2024-05-30 15:12:25.947303908 +0800 CST m=+5.756771529 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:14:25.972 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:12:25.978 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:02:25.978 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.cli.28558.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450115181185073153 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115181185073153 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:12:32 CST 2024] <<<<<< START cdc server in canal_json_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.2858328585.out server --log-file /tmp/tidb_cdc_test/canal_json_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_handle_key_only/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:12:35 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c3cfcc9-e84e-4219-95a8-7eb7629f16dd {"id":"5c3cfcc9-e84e-4219-95a8-7eb7629f16dd","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8579a75c6 5c3cfcc9-e84e-4219-95a8-7eb7629f16dd /tidb/cdc/default/default/upstream/7374687065881145850 {"id":7374687065881145850,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c3cfcc9-e84e-4219-95a8-7eb7629f16dd {"id":"5c3cfcc9-e84e-4219-95a8-7eb7629f16dd","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8579a75c6 5c3cfcc9-e84e-4219-95a8-7eb7629f16dd /tidb/cdc/default/default/upstream/7374687065881145850 {"id":7374687065881145850,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c3cfcc9-e84e-4219-95a8-7eb7629f16dd {"id":"5c3cfcc9-e84e-4219-95a8-7eb7629f16dd","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8579a75c6 5c3cfcc9-e84e-4219-95a8-7eb7629f16dd /tidb/cdc/default/default/upstream/7374687065881145850 {"id":7374687065881145850,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.cli.28629.out cli changefeed create --start-ts=450115181185073153 '--sink-uri=kafka://127.0.0.1:9092/canal-json-handle-key-only?protocol=canal-json&enable-tidb-extension=true&max-message-bytes=1000&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/conf/changefeed.toml Create changefeed successfully! ID: 0107b424-547d-41ff-8ae3-ae33cfcdb7d8 Info: {"upstream_id":7374687065881145850,"namespace":"default","id":"0107b424-547d-41ff-8ae3-ae33cfcdb7d8","sink_uri":"kafka://127.0.0.1:9092/canal-json-handle-key-only?protocol=canal-json\u0026enable-tidb-extension=true\u0026max-message-bytes=1000\u0026kafka-version=2.4.1","create_time":"2024-05-30T15:12:35.908102217+08:00","start_ts":450115181185073153,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"snappy","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115181185073153,"checkpoint_ts":450115181185073153,"checkpoint_time":"2024-05-30 15:12:30.883"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Thu May 30 15:12:43 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh: line 1: 28669 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/open_protocol_handle_key_only Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2161a7a40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:29617, start at 2024-05-30 15:13:08.605644666 +0800 CST m=+5.807299149 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:15:08.616 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:13:08.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:03:08.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2161a7a40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:29617, start at 2024-05-30 15:13:08.605644666 +0800 CST m=+5.807299149 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:15:08.616 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:13:08.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:03:08.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2161a9f40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:29696, start at 2024-05-30 15:13:08.764891094 +0800 CST m=+5.864245582 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:15:08.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:13:08.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:03:08.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.cli.30977.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450115192352931841 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115192352931841 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:13:14 CST 2024] <<<<<< START cdc server in open_protocol_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.3100331005.out server --log-file /tmp/tidb_cdc_test/open_protocol_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/open_protocol_handle_key_only/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:13:18 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09f66fbc-f81c-41df-8c25-2d2d0f0b2ea1 {"id":"09f66fbc-f81c-41df-8c25-2d2d0f0b2ea1","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8584337c3 09f66fbc-f81c-41df-8c25-2d2d0f0b2ea1 /tidb/cdc/default/default/upstream/7374687253503717150 {"id":7374687253503717150,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09f66fbc-f81c-41df-8c25-2d2d0f0b2ea1 {"id":"09f66fbc-f81c-41df-8c25-2d2d0f0b2ea1","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8584337c3 09f66fbc-f81c-41df-8c25-2d2d0f0b2ea1 /tidb/cdc/default/default/upstream/7374687253503717150 {"id":7374687253503717150,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09f66fbc-f81c-41df-8c25-2d2d0f0b2ea1 {"id":"09f66fbc-f81c-41df-8c25-2d2d0f0b2ea1","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8584337c3 09f66fbc-f81c-41df-8c25-2d2d0f0b2ea1 /tidb/cdc/default/default/upstream/7374687253503717150 {"id":7374687253503717150,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.cli.31051.out cli changefeed create --start-ts=450115192352931841 '--sink-uri=kafka://127.0.0.1:9092/open-protocol-handle-key-only?protocol=open-protocol&max-message-bytes=800&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/conf/changefeed.toml Create changefeed successfully! ID: 481a00fe-9a48-40c4-af5d-729cd504e963 Info: {"upstream_id":7374687253503717150,"namespace":"default","id":"481a00fe-9a48-40c4-af5d-729cd504e963","sink_uri":"kafka://127.0.0.1:9092/open-protocol-handle-key-only?protocol=open-protocol\u0026max-message-bytes=800\u0026kafka-version=2.4.1","create_time":"2024-05-30T15:13:18.530657743+08:00","start_ts":450115192352931841,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115192352931841,"checkpoint_ts":450115192352931841,"checkpoint_time":"2024-05-30 15:13:13.485"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:13:28 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh: line 1: 31092 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/canal_json_claim_check Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216476900010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:32130, start at 2024-05-30 15:13:54.615795861 +0800 CST m=+5.811747748 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:15:54.623 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:13:54.596 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:03:54.596 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216476900010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:32130, start at 2024-05-30 15:13:54.615795861 +0800 CST m=+5.811747748 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:15:54.623 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:13:54.596 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:03:54.596 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216477400014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:32213, start at 2024-05-30 15:13:54.686422629 +0800 CST m=+5.713581123 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:15:54.694 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:13:54.690 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:03:54.690 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.cli.33439.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450115204440915969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115204440915969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:14:01 CST 2024] <<<<<< START cdc server in canal_json_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.3346533467.out server --log-file /tmp/tidb_cdc_test/canal_json_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_claim_check/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:14:04 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2208c536-bf17-422b-931a-5177253bd755 {"id":"2208c536-bf17-422b-931a-5177253bd755","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc858f118c1 2208c536-bf17-422b-931a-5177253bd755 /tidb/cdc/default/default/upstream/7374687450001205641 {"id":7374687450001205641,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2208c536-bf17-422b-931a-5177253bd755 {"id":"2208c536-bf17-422b-931a-5177253bd755","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc858f118c1 2208c536-bf17-422b-931a-5177253bd755 /tidb/cdc/default/default/upstream/7374687450001205641 {"id":7374687450001205641,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2208c536-bf17-422b-931a-5177253bd755 {"id":"2208c536-bf17-422b-931a-5177253bd755","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc858f118c1 2208c536-bf17-422b-931a-5177253bd755 /tidb/cdc/default/default/upstream/7374687450001205641 {"id":7374687450001205641,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.cli.33512.out cli changefeed create --start-ts=450115204440915969 '--sink-uri=kafka://127.0.0.1:9092/canal-json-claim-check?protocol=canal-json&enable-tidb-extension=true&max-message-bytes=1000&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/conf/changefeed.toml Create changefeed successfully! ID: 27c4f3e5-15a6-4dab-8fa2-9281f226b8a8 Info: {"upstream_id":7374687450001205641,"namespace":"default","id":"27c4f3e5-15a6-4dab-8fa2-9281f226b8a8","sink_uri":"kafka://127.0.0.1:9092/canal-json-claim-check?protocol=canal-json\u0026enable-tidb-extension=true\u0026max-message-bytes=1000\u0026kafka-version=2.4.1","create_time":"2024-05-30T15:14:04.62567444+08:00","start_ts":450115204440915969,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/canal-json-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115204440915969,"checkpoint_ts":450115204440915969,"checkpoint_time":"2024-05-30 15:13:59.597"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:14:12 CST 2024] <<<<<< run test case canal_json_claim_check success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh: line 1: 33544 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/open_protocol_claim_check Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2167302c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:34541, start at 2024-05-30 15:14:39.286297315 +0800 CST m=+6.147670106 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:16:39.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:14:39.293 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:04:39.293 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2167302c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:34541, start at 2024-05-30 15:14:39.286297315 +0800 CST m=+6.147670106 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:16:39.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:14:39.293 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:04:39.293 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216730200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:34627, start at 2024-05-30 15:14:39.289503741 +0800 CST m=+5.999127095 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:16:39.298 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:14:39.290 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:04:39.290 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.35851.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450115216065953793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115216065953793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2723 0 --:--:-- --:--:-- --:--:-- 2728 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-30 15:14:36.208","puller_resolved_ts":"2024-05-30 15:14:28.208","last_synced_ts":"2024-05-30 15:12:19.658","now_ts":"2024-05-30 15:14:36.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:14:36.208","puller_resolved_ts":"2024-05-30' '15:14:28.208","last_synced_ts":"2024-05-30' '15:12:19.658","now_ts":"2024-05-30' '15:14:36.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:14:36.208","puller_resolved_ts":"2024-05-30' '15:14:28.208","last_synced_ts":"2024-05-30' '15:12:19.658","now_ts":"2024-05-30' '15:14:36.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.35895.out cli tso query --pd=http://127.0.0.1:2379 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... + set +x + tso='450115217455316997 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115217455316997 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Thu May 30 15:14:50 CST 2024] <<<<<< START cdc server in open_protocol_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.3591935921.out server --log-file /tmp/tidb_cdc_test/open_protocol_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/open_protocol_claim_check/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:14:53 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e1ea249-4697-4922-ae59-ba5924115c7b {"id":"7e1ea249-4697-4922-ae59-ba5924115c7b","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8599e531e 7e1ea249-4697-4922-ae59-ba5924115c7b /tidb/cdc/default/default/upstream/7374687642860285683 {"id":7374687642860285683,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e1ea249-4697-4922-ae59-ba5924115c7b {"id":"7e1ea249-4697-4922-ae59-ba5924115c7b","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8599e531e 7e1ea249-4697-4922-ae59-ba5924115c7b /tidb/cdc/default/default/upstream/7374687642860285683 {"id":7374687642860285683,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e1ea249-4697-4922-ae59-ba5924115c7b {"id":"7e1ea249-4697-4922-ae59-ba5924115c7b","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8599e531e 7e1ea249-4697-4922-ae59-ba5924115c7b /tidb/cdc/default/default/upstream/7374687642860285683 {"id":7374687642860285683,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.35970.out cli changefeed create --start-ts=450115216065953793 --target-ts=450115217455316997 '--sink-uri=kafka://127.0.0.1:9092/open-protocol-claim-check?protocol=open-protocol&max-message-bytes=800&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/conf/changefeed.toml Create changefeed successfully! ID: 73bb2b9c-563a-490b-9d2b-bc82fe37a160 Info: {"upstream_id":7374687642860285683,"namespace":"default","id":"73bb2b9c-563a-490b-9d2b-bc82fe37a160","sink_uri":"kafka://127.0.0.1:9092/open-protocol-claim-check?protocol=open-protocol\u0026max-message-bytes=800\u0026kafka-version=2.4.1","create_time":"2024-05-30T15:14:54.224733114+08:00","start_ts":450115216065953793,"target_ts":450115217455316997,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"lz4","claim_check_storage_uri":"file:///tmp/open-protocol-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115216065953793,"checkpoint_ts":450115216065953793,"checkpoint_time":"2024-05-30 15:14:43.943"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x table test.finish_mark not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:15:01 CST 2024] <<<<<< run test case open_protocol_claim_check success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216895cc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:22944, start at 2024-05-30 15:15:02.137440211 +0800 CST m=+5.907559526 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:17:02.145 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:15:02.131 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:05:02.131 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216895cc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:22944, start at 2024-05-30 15:15:02.137440211 +0800 CST m=+5.907559526 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:17:02.145 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:15:02.131 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:05:02.131 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216896840013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:23019, start at 2024-05-30 15:15:02.200499455 +0800 CST m=+5.822647326 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:17:02.209 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:15:02.177 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:05:02.177 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.24236.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450115222131441665 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115222131441665 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450115222131441665 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Thu May 30 15:15:08 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2427124273.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:15:11 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3fdd03f-cfdb-4b6e-9e87-8ce7e1d58368 {"id":"c3fdd03f-cfdb-4b6e-9e87-8ce7e1d58368","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc859f68cc2 c3fdd03f-cfdb-4b6e-9e87-8ce7e1d58368 /tidb/cdc/default/default/upstream/7374687739781874698 {"id":7374687739781874698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3fdd03f-cfdb-4b6e-9e87-8ce7e1d58368 {"id":"c3fdd03f-cfdb-4b6e-9e87-8ce7e1d58368","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc859f68cc2 c3fdd03f-cfdb-4b6e-9e87-8ce7e1d58368 /tidb/cdc/default/default/upstream/7374687739781874698 {"id":7374687739781874698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3fdd03f-cfdb-4b6e-9e87-8ce7e1d58368 {"id":"c3fdd03f-cfdb-4b6e-9e87-8ce7e1d58368","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc859f68cc2 c3fdd03f-cfdb-4b6e-9e87-8ce7e1d58368 /tidb/cdc/default/default/upstream/7374687739781874698 {"id":7374687739781874698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450115222131441665 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.24318.out cli changefeed create --start-ts=450115222131441665 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374687739781874698,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-30T15:15:12.048026796+08:00","start_ts":450115222131441665,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115222131441665,"checkpoint_ts":450115222131441665,"checkpoint_time":"2024-05-30 15:15:07.081"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh: line 1: 36002 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:15:13 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: kafka... <<================= [Thu May 30 15:15:16 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D432E62D400103 < X-Xss-Protection: 1; mode=block < Date: Thu, 30 May 2024 07:15:22 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 6000 0 --:--:-- --:--:-- --:--:-- 6025 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-30 15:15:07.081","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-30 15:15:33.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:15:07.081","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:15:33.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:15:07.081","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:15:33.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216abbac0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:37182, start at 2024-05-30 15:15:37.365241745 +0800 CST m=+5.851853899 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:17:37.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:15:37.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:05:37.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216abbac0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:37182, start at 2024-05-30 15:15:37.365241745 +0800 CST m=+5.851853899 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:17:37.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:15:37.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:05:37.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216abdb80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-cnb3v, pid:37261, start at 2024-05-30 15:15:37.458959958 +0800 CST m=+5.854197669 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:17:37.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:15:37.454 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:05:37.454 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Thu May 30 15:15:42 CST 2024] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.3858038582.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:15:45 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fe732eb-c5a4-47a4-9daf-19b3c309910a {"id":"8fe732eb-c5a4-47a4-9daf-19b3c309910a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85a825fbe 8fe732eb-c5a4-47a4-9daf-19b3c309910a /tidb/cdc/default/default/upstream/7374687891087930806 {"id":7374687891087930806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fe732eb-c5a4-47a4-9daf-19b3c309910a {"id":"8fe732eb-c5a4-47a4-9daf-19b3c309910a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85a825fbe 8fe732eb-c5a4-47a4-9daf-19b3c309910a /tidb/cdc/default/default/upstream/7374687891087930806 {"id":7374687891087930806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8fe732eb-c5a4-47a4-9daf-19b3c309910a {"id":"8fe732eb-c5a4-47a4-9daf-19b3c309910a","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85a825fbe 8fe732eb-c5a4-47a4-9daf-19b3c309910a /tidb/cdc/default/default/upstream/7374687891087930806 {"id":7374687891087930806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7374687891087930806,"namespace":"default","id":"test-normal","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-normal-4073?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:15:45.420736088+08:00","start_ts":450115231304384513,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115231304384513,"checkpoint_ts":450115231304384513,"checkpoint_time":"2024-05-30 15:15:42.073"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7374687891087930806,"namespace":"default","id":"test-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-1-16145?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:15:45.619218356+08:00","start_ts":450115231304384513,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115231304384513,"checkpoint_ts":450115231304384513,"checkpoint_time":"2024-05-30 15:15:42.073"} + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7374687891087930806,"namespace":"default","id":"test-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-2-9528?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-30T15:15:45.841117461+08:00","start_ts":450115231304384513,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115231304384513,"checkpoint_ts":450115231304384513,"checkpoint_time":"2024-05-30 15:15:42.073"} [Thu May 30 15:15:45 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> [Thu May 30 15:15:45 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> [Thu May 30 15:15:45 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... table multi_tables_ddl_test.t55 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table multi_tables_ddl_test.t55 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216c003c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:25213, start at 2024-05-30 15:15:58.115881003 +0800 CST m=+6.010174050 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:17:58.124 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:15:58.095 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:05:58.095 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216c003c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:25213, start at 2024-05-30 15:15:58.115881003 +0800 CST m=+6.010174050 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:17:58.124 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:15:58.095 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:05:58.095 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f216c01c4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:25283, start at 2024-05-30 15:15:58.207752841 +0800 CST m=+5.936046972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:17:58.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:15:58.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:05:58.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7374687891087930806, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450115233204928535, "checkpoint_time": "2024-05-30 15:15:49.323", "error": null }' + echo '{ "upstream_id": 7374687891087930806, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450115233204928535, "checkpoint_time": "2024-05-30 15:15:49.323", "error": null }' { "upstream_id": 7374687891087930806, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450115233204928535, "checkpoint_time": "2024-05-30 15:15:49.323", "error": null } ++ echo '{' '"upstream_id":' 7374687891087930806, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450115233204928535, '"checkpoint_time":' '"2024-05-30' '15:15:49.323",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374687891087930806, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450115233204928535, '"checkpoint_time":' '"2024-05-30' '15:15:49.323",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7374687891087930806, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450115236416192525, "checkpoint_time": "2024-05-30 15:16:01.573", "error": null }' + echo '{ "upstream_id": 7374687891087930806, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450115236416192525, "checkpoint_time": "2024-05-30 15:16:01.573", "error": null }' { "upstream_id": 7374687891087930806, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450115236416192525, "checkpoint_time": "2024-05-30 15:16:01.573", "error": null } ++ echo '{' '"upstream_id":' 7374687891087930806, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450115236416192525, '"checkpoint_time":' '"2024-05-30' '15:16:01.573",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7374687891087930806, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450115236416192525, '"checkpoint_time":' '"2024-05-30' '15:16:01.573",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7374687891087930806, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450115232680902664, "checkpoint_time": "2024-05-30 15:15:47.324", "error": { "time": "2024-05-30T15:15:49.753181786+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7374687891087930806, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450115232680902664, "checkpoint_time": "2024-05-30 15:15:47.324", "error": { "time": "2024-05-30T15:15:49.753181786+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7374687891087930806, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450115232680902664, "checkpoint_time": "2024-05-30 15:15:47.324", "error": { "time": "2024-05-30T15:15:49.753181786+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7374687891087930806, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450115232680902664, '"checkpoint_time":' '"2024-05-30' '15:15:47.324",' '"error":' '{' '"time":' '"2024-05-30T15:15:49.753181786+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7374687891087930806, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450115232680902664, '"checkpoint_time":' '"2024-05-30' '15:15:47.324",' '"error":' '{' '"time":' '"2024-05-30T15:15:49.753181786+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully wait process cdc.test exit for 1-th time... + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26562.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... + set +x + tso='450115236775854081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115236775854081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450115236775854081 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Thu May 30 15:16:04 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2659526597.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Thu May 30 15:16:04 CST 2024] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:16:07 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2c1f5eb4-60fc-45dc-8f8e-f16f425cf3fc {"id":"2c1f5eb4-60fc-45dc-8f8e-f16f425cf3fc","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85ada88c7 2c1f5eb4-60fc-45dc-8f8e-f16f425cf3fc /tidb/cdc/default/default/upstream/7374687977349066674 {"id":7374687977349066674,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2c1f5eb4-60fc-45dc-8f8e-f16f425cf3fc {"id":"2c1f5eb4-60fc-45dc-8f8e-f16f425cf3fc","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85ada88c7 2c1f5eb4-60fc-45dc-8f8e-f16f425cf3fc /tidb/cdc/default/default/upstream/7374687977349066674 {"id":7374687977349066674,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2c1f5eb4-60fc-45dc-8f8e-f16f425cf3fc {"id":"2c1f5eb4-60fc-45dc-8f8e-f16f425cf3fc","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85ada88c7 2c1f5eb4-60fc-45dc-8f8e-f16f425cf3fc /tidb/cdc/default/default/upstream/7374687977349066674 {"id":7374687977349066674,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450115236775854081 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26638.out cli changefeed create --start-ts=450115236775854081 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374687977349066674,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-30T15:16:07.927685669+08:00","start_ts":450115236775854081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115236775854081,"checkpoint_ts":450115236775854081,"checkpoint_time":"2024-05-30 15:16:02.945"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2001 0 --:--:-- --:--:-- --:--:-- 2009 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-30 15:16:02.945","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-30 15:16:09.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:16:02.945","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:16:09.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:16:02.945","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:16:09.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-30 15:16:02.945' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:16:02.945","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:16:09.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:16:02.945","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:16:09.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-30 15:16:09' + echo 'sink_checkpoint_ts is 2024-05-30' 15:16:02.945 sink_checkpoint_ts is 2024-05-30 15:16:02.945 ++ date -d '2024-05-30 15:16:02.945' +%s + checkpoint_timestamp=1717053362 ++ date -d '2024-05-30 15:16:09' +%s + current_timestamp=1717053369 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2457 0 --:--:-- --:--:-- --:--:-- 2454 100 243 100 243 0 0 2455 0 --:--:-- --:--:-- --:--:-- 2454 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-30 15:16:15.296","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-30 15:16:10.146","now_ts":"2024-05-30 15:16:17.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:16:15.296","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:16:10.146","now_ts":"2024-05-30' '15:16:17.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:16:15.296","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:16:10.146","now_ts":"2024-05-30' '15:16:17.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2722 0 --:--:-- --:--:-- --:--:-- 2728 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-30 15:18:25.396","puller_resolved_ts":"2024-05-30 15:18:18.395","last_synced_ts":"2024-05-30 15:16:10.146","now_ts":"2024-05-30 15:18:27.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:18:25.396","puller_resolved_ts":"2024-05-30' '15:18:18.395","last_synced_ts":"2024-05-30' '15:16:10.146","now_ts":"2024-05-30' '15:18:27.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 24581 10.5 0.0 13691880 147716 ? Sl 15:15 0:16 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 24641 6.6 0.0 13872544 137200 ? Sl 15:15 0:10 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1717053513.156495,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002272e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717053513.156589,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717053513.247258,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000b46700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1717053513.2473385,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717053513.4651814,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002274380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717053513.4653575,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-30T15:18:37.205718+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f98380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:18:37.212304+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d4e380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:18:37.389464+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00118f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:18:43.206847+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f98380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:18:43.213982+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d4e380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:18:43.391013+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00118f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:18:49.207333+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f98380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:18:49.215843+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d4e380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:18:49.392354+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00118f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:18:55.208532+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f98380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:18:55.216865+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d4e380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:18:55.393698+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00118f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:18:57.196892+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f98380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-30T15:18:57.196968+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-30T15:18:57.202777+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d4e380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-30T15:18:57.202828+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-30T15:18:57.380365+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00118f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-30T15:18:57.380432+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:19:01.210038+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f98380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:19:01.21827+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d4e380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:19:01.395136+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00118f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:19:07.211147+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f98380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:19:07.219243+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d4e380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:19:07.396573+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00118f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":1717053548.1584516,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002272e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717053548.1585147,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717053548.2483032,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000b46700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1717053548.2483609,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717053548.4669912,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002274380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717053548.4670513,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0{"level":"warn","ts":"2024-05-30T15:19:13.212453+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f98380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:19:13.220619+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d4e380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:19:13.397327+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00118f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... {"level":"warn","ts":"2024-05-30T15:19:19.213765+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f98380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:19:19.221878+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000d4e380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-30T15:19:19.399221+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00118f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2179ac800002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:27757, start at 2024-05-30 15:19:42.114153292 +0800 CST m=+5.934576733 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:21:42.122 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:19:42.112 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:09:42.112 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2179ac800002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:27757, start at 2024-05-30 15:19:42.114153292 +0800 CST m=+5.934576733 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:21:42.122 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:19:42.112 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:09:42.112 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2179aeb00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:27829, start at 2024-05-30 15:19:42.273633678 +0800 CST m=+5.973862643 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:21:42.282 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:19:42.252 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:09:42.252 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.29089.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450115295487459329 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115295487459329 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450115295487459329 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Thu May 30 15:19:48 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2912429126.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:19:51 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fad6dc79-3943-4931-8b2e-5192978d93bf {"id":"fad6dc79-3943-4931-8b2e-5192978d93bf","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85e3dc7c5 fad6dc79-3943-4931-8b2e-5192978d93bf /tidb/cdc/default/default/upstream/7374688935151856857 {"id":7374688935151856857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fad6dc79-3943-4931-8b2e-5192978d93bf {"id":"fad6dc79-3943-4931-8b2e-5192978d93bf","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85e3dc7c5 fad6dc79-3943-4931-8b2e-5192978d93bf /tidb/cdc/default/default/upstream/7374688935151856857 {"id":7374688935151856857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fad6dc79-3943-4931-8b2e-5192978d93bf {"id":"fad6dc79-3943-4931-8b2e-5192978d93bf","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc85e3dc7c5 fad6dc79-3943-4931-8b2e-5192978d93bf /tidb/cdc/default/default/upstream/7374688935151856857 {"id":7374688935151856857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450115295487459329 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.29173.out cli changefeed create --start-ts=450115295487459329 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374688935151856857,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-30T15:19:51.875420868+08:00","start_ts":450115295487459329,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115295487459329,"checkpoint_ts":450115295487459329,"checkpoint_time":"2024-05-30 15:19:46.912"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 27237 26.1 0.4 3710960 1727144 ? Sl 15:19 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 27238 17.5 0.4 3689968 1685152 ? Sl 15:19 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 27239 16.8 0.4 3685356 1674620 ? Sl 15:19 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 27241 24.7 0.4 3695088 1721204 ? Sl 15:19 0:06 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3334 0 --:--:-- --:--:-- --:--:-- 3375 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-30 15:19:59.262","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-30 15:19:53.862","now_ts":"2024-05-30 15:20:00.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:19:59.262","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:19:53.862","now_ts":"2024-05-30' '15:20:00.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:19:59.262","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:19:53.862","now_ts":"2024-05-30' '15:20:00.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 7393 0 --:--:-- --:--:-- --:--:-- 7453 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-30 15:20:00.262","puller_resolved_ts":"2024-05-30 15:20:00.262","last_synced_ts":"2024-05-30 15:19:53.862","now_ts":"2024-05-30 15:22:11.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:20:00.262","puller_resolved_ts":"2024-05-30' '15:20:00.262","last_synced_ts":"2024-05-30' '15:19:53.862","now_ts":"2024-05-30' '15:22:11.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:20:00.262","puller_resolved_ts":"2024-05-30' '15:20:00.262","last_synced_ts":"2024-05-30' '15:19:53.862","now_ts":"2024-05-30' '15:22:11.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21844610000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:30275, start at 2024-05-30 15:22:35.797131801 +0800 CST m=+5.777732154 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:24:35.806 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:22:35.780 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:12:35.780 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f21844610000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:30275, start at 2024-05-30 15:22:35.797131801 +0800 CST m=+5.777732154 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:24:35.806 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:22:35.780 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:12:35.780 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f2184487c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:30361, start at 2024-05-30 15:22:35.9729386 +0800 CST m=+5.795602058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:24:35.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:22:35.985 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:12:35.985 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.31595.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450115341079019521 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115341079019521 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450115341079019521 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Thu May 30 15:22:42 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3162831630.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:22:45 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e350307-6abd-4e9b-89cb-6ea5c1803b07 {"id":"1e350307-6abd-4e9b-89cb-6ea5c1803b07","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc860e303c8 1e350307-6abd-4e9b-89cb-6ea5c1803b07 /tidb/cdc/default/default/upstream/7374689679558136019 {"id":7374689679558136019,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e350307-6abd-4e9b-89cb-6ea5c1803b07 {"id":"1e350307-6abd-4e9b-89cb-6ea5c1803b07","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc860e303c8 1e350307-6abd-4e9b-89cb-6ea5c1803b07 /tidb/cdc/default/default/upstream/7374689679558136019 {"id":7374689679558136019,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e350307-6abd-4e9b-89cb-6ea5c1803b07 {"id":"1e350307-6abd-4e9b-89cb-6ea5c1803b07","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc860e303c8 1e350307-6abd-4e9b-89cb-6ea5c1803b07 /tidb/cdc/default/default/upstream/7374689679558136019 {"id":7374689679558136019,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450115341079019521 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.31681.out cli changefeed create --start-ts=450115341079019521 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374689679558136019,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-30T15:22:45.822886778+08:00","start_ts":450115341079019521,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115341079019521,"checkpoint_ts":450115341079019521,"checkpoint_time":"2024-05-30 15:22:40.830"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 30275 12.0 0.0 2386596 218420 ? Sl 15:22 0:03 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717053750013258539.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 30279 4.0 0.0 2524680 162036 ? Sl 15:22 0:01 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717053750016224294.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 30361 12.1 0.0 2503264 235532 ? Sl 15:22 0:03 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717053750170433741.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 1719 0 --:--:-- --:--:-- --:--:-- 1723 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-30 15:22:53.180","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-30 15:22:47.380","now_ts":"2024-05-30 15:22:54.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:22:53.180","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:22:47.380","now_ts":"2024-05-30' '15:22:54.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:22:53.180","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-30' '15:22:47.380","now_ts":"2024-05-30' '15:22:54.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2745 0 --:--:-- --:--:-- --:--:-- 2762 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-30 15:25:03.330","puller_resolved_ts":"2024-05-30 15:24:56.330","last_synced_ts":"2024-05-30 15:22:47.380","now_ts":"2024-05-30 15:25:04.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:25:03.330","puller_resolved_ts":"2024-05-30' '15:24:56.330","last_synced_ts":"2024-05-30' '15:22:47.380","now_ts":"2024-05-30' '15:25:04.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-30' '15:25:03.330","puller_resolved_ts":"2024-05-30' '15:24:56.330","last_synced_ts":"2024-05-30' '15:22:47.380","now_ts":"2024-05-30' '15:25:04.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Starting Downstream PD... Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f218ec8e80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:32718, start at 2024-05-30 15:25:28.029206222 +0800 CST m=+5.908277971 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:27:28.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:25:28.044 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:15:28.044 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f218ec8e80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:32718, start at 2024-05-30 15:25:28.029206222 +0800 CST m=+5.908277971 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:27:28.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:25:28.044 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:15:28.044 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f218ecb400014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-727-f5787, pid:32792, start at 2024-05-30 15:25:28.18741192 +0800 CST m=+5.911470648 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240530-15:27:28.196 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240530-15:25:28.194 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240530-15:15:28.194 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-26-gdb0862135 Edition: Community Git Commit Hash: db0862135d4a94904069a37ec50c69cfe6099952 Git Branch: HEAD UTC Build Time: 2024-05-27 08:18:38 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-05-27 08:24:01 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-26-gdb0862135"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["db0862135d4a94904069a37ec50c69cfe6099952"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.34073.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450115386184564737 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450115386184564737 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450115386184564737 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Thu May 30 15:25:34 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3409634098.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Thu, 30 May 2024 07:25:37 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/360cb96f-a04a-4993-9fc2-4784f66d2a18 {"id":"360cb96f-a04a-4993-9fc2-4784f66d2a18","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8638930c2 360cb96f-a04a-4993-9fc2-4784f66d2a18 /tidb/cdc/default/default/upstream/7374690426835767621 {"id":7374690426835767621,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/360cb96f-a04a-4993-9fc2-4784f66d2a18 {"id":"360cb96f-a04a-4993-9fc2-4784f66d2a18","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8638930c2 360cb96f-a04a-4993-9fc2-4784f66d2a18 /tidb/cdc/default/default/upstream/7374690426835767621 {"id":7374690426835767621,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/360cb96f-a04a-4993-9fc2-4784f66d2a18 {"id":"360cb96f-a04a-4993-9fc2-4784f66d2a18","address":"127.0.0.1:8300","version":"v7.5.1-58-g8f92142d7"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fc8638930c2 360cb96f-a04a-4993-9fc2-4784f66d2a18 /tidb/cdc/default/default/upstream/7374690426835767621 {"id":7374690426835767621,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450115386184564737 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.34140.out cli changefeed create --start-ts=450115386184564737 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7374690426835767621,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-30T15:25:37.894931042+08:00","start_ts":450115386184564737,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-58-g8f92142d7","resolved_ts":450115386184564737,"checkpoint_ts":450115386184564737,"checkpoint_time":"2024-05-30 15:25:32.894"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 8806 0 --:--:-- --:--:-- --:--:-- 8817 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-30 15:25:32.894","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-30 15:25:59.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:25:32.894","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:25:59.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-30' '15:25:32.894","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-30' '15:25:59.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Thu May 30 15:26:10 CST 2024] <<<<<< run test case synced_status success! >>>>>>' [Thu May 30 15:26:10 CST 2024] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-727/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS